diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/debug-internal.log" @@ -0,0 +1,35052 @@ +2024-09-22 06:33:27,563 INFO StreamThr :1340627 [internal.py:wandb_internal():85] W&B internal server running at pid: 1340627, started at: 2024-09-22 06:33:27.562805 +2024-09-22 06:33:27,565 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status +2024-09-22 06:33:27,566 INFO WriterThread:1340627 [datastore.py:open_for_write():87] open: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/run-rptdqsqq.wandb +2024-09-22 06:33:27,568 DEBUG SenderThread:1340627 [sender.py:send():379] send: header +2024-09-22 06:33:27,588 DEBUG SenderThread:1340627 [sender.py:send():379] send: run +2024-09-22 06:33:28,004 INFO SenderThread:1340627 [dir_watcher.py:__init__():211] watching files in: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files +2024-09-22 06:33:28,005 INFO SenderThread:1340627 [sender.py:_start_run_threads():1188] run started: rptdqsqq with start time 1726958007.564867 +2024-09-22 06:33:28,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: check_version +2024-09-22 06:33:28,009 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: check_version +2024-09-22 06:33:28,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: run_start +2024-09-22 06:33:28,274 DEBUG HandlerThread:1340627 [system_info.py:__init__():26] System info init +2024-09-22 06:33:28,274 DEBUG HandlerThread:1340627 [system_info.py:__init__():41] System info init done +2024-09-22 06:33:28,275 INFO HandlerThread:1340627 [system_monitor.py:start():194] Starting system monitor +2024-09-22 06:33:28,275 INFO SystemMonitor:1340627 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-22 06:33:28,275 INFO HandlerThread:1340627 [system_monitor.py:probe():214] Collecting system info +2024-09-22 06:33:28,275 INFO SystemMonitor:1340627 [interfaces.py:start():188] Started cpu monitoring +2024-09-22 06:33:28,276 INFO SystemMonitor:1340627 [interfaces.py:start():188] Started disk monitoring +2024-09-22 06:33:28,277 INFO SystemMonitor:1340627 [interfaces.py:start():188] Started gpu monitoring +2024-09-22 06:33:28,277 INFO SystemMonitor:1340627 [interfaces.py:start():188] Started memory monitoring +2024-09-22 06:33:28,278 INFO SystemMonitor:1340627 [interfaces.py:start():188] Started network monitoring +2024-09-22 06:33:28,392 DEBUG HandlerThread:1340627 [system_info.py:probe():152] Probing system +2024-09-22 06:33:28,395 DEBUG HandlerThread:1340627 [system_info.py:_probe_git():137] Probing git +2024-09-22 06:33:28,408 DEBUG HandlerThread:1340627 [system_info.py:_probe_git():145] Probing git done +2024-09-22 06:33:28,408 DEBUG HandlerThread:1340627 [system_info.py:probe():200] Probing system done +2024-09-22 06:33:28,408 DEBUG HandlerThread:1340627 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35', 'python': '3.11.9', 'heartbeatAt': '2024-09-21T22:33:28.392138', 'startedAt': '2024-09-21T22:33:27.553001', 'docker': None, 'cuda': None, 'args': ('--local_rank=0', '--model_name_or_path', '/aifs4su/yaodong/models/Qwen2-VL-7B-Instruct', '--train_datasets', '/aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted', '--output_dir', '../outputs/dpo_tiv2t_10k_baseline', '--train_data_files', 'extracted_preference_10k_washed.json', '--train_template', 'NExTQA_preference', '--train_split', 'train', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '1', '--gradient_accumulation_steps', '1'), 'state': 'running', 'program': '-m align_anything.trainers.tiv_to_t.dpo', 'codePathLocal': None, 'git': {'remote': 'git@github.com-hantao:htlou/align-anything.git', 'commit': '9eff99b0ee34f7265b8a2e1824650634c898f654'}, 'email': '2200017789@stu.pku.edu.cn', 'root': '/aifs4su/yaodong/projects/hantao/dev_cham/align-anything', 'host': 'dgx-081', 'username': 'yangyaodong', 'executable': '/aifs4su/yaodong/miniconda3/envs/hantao_tiv/bin/python3.11', 'cpu_count': 112, 'cpu_count_logical': 224, 'cpu_freq': {'current': 2064.275857142857, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3799.998, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3799.998, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.001, 'min': 800.0, 'max': 3800.0}, {'current': 3799.998, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3799.993, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3799.907, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 1758.8549690246582, 'used': 58.09877014160156}}, 'gpu': 'NVIDIA H800', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}], 'memory': {'total': 2015.5615730285645}} +2024-09-22 06:33:28,408 INFO HandlerThread:1340627 [system_monitor.py:probe():224] Finished collecting system info +2024-09-22 06:33:28,408 INFO HandlerThread:1340627 [system_monitor.py:probe():227] Publishing system info +2024-09-22 06:33:28,408 DEBUG HandlerThread:1340627 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment +2024-09-22 06:33:29,006 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/conda-environment.yaml +2024-09-22 06:33:33,885 DEBUG HandlerThread:1340627 [system_info.py:_save_conda():224] Saving conda packages done +2024-09-22 06:33:33,888 INFO HandlerThread:1340627 [system_monitor.py:probe():229] Finished publishing system info +2024-09-22 06:33:33,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:33:33,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 06:33:33,900 DEBUG SenderThread:1340627 [sender.py:send():379] send: files +2024-09-22 06:33:33,900 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-09-22 06:33:34,010 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/conda-environment.yaml +2024-09-22 06:33:34,010 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-metadata.json +2024-09-22 06:33:34,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: python_packages +2024-09-22 06:33:34,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:33:34,309 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: python_packages +2024-09-22 06:33:34,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:34,311 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:33:34,544 DEBUG SenderThread:1340627 [sender.py:send():379] send: telemetry +2024-09-22 06:33:34,726 INFO wandb-upload_0:1340627 [upload_job.py:push():130] Uploaded file /tmp/tmpgm4ub6rmwandb/ompu2pe5-wandb-metadata.json +2024-09-22 06:33:35,011 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/requirements.txt +2024-09-22 06:33:35,012 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:33:35,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:36,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:37,013 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:33:37,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:38,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:38,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:33:39,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:40,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:41,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:42,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:43,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:44,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:33:44,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:45,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:46,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:47,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:48,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:49,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:33:49,310 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:33:49,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:49,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:33:50,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:51,024 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:33:51,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:52,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:53,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:54,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:55,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:33:55,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:56,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:57,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:57,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:33:57,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: telemetry +2024-09-22 06:33:58,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:33:59,031 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:34:00,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:01,033 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/config.yaml +2024-09-22 06:34:05,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:10,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:14,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:34:14,779 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:34:14,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:15,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:16,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:16,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:17,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:34:17,219 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:34:17,219 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:34:17,221 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:34:17,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:18,049 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:34:18,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:19,050 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:34:19,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:20,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:21,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:21,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:22,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:23,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:24,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:25,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:26,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:26,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:28,279 DEBUG SystemMonitor:1340627 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-22 06:34:28,281 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:34:28,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:29,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:34:29,778 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:34:29,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:30,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:34:30,383 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:34:30,383 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:34:30,385 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:34:30,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:31,062 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:34:31,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:32,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:33,064 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:34:37,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:42,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:47,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:50,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:50,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:34:50,182 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:34:51,179 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:51,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:34:51,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:34:51,815 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:34:51,817 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:34:52,082 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:34:52,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:52,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:53,083 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:34:53,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:54,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:55,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:56,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:57,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:58,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:34:58,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:34:58,283 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:35:02,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:03,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:03,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:35:04,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:35:04,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:35:04,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:35:04,341 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:35:04,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:05,111 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:35:05,179 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:35:05,180 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:35:07,127 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:35:09,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:35:14,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:35:19,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:35:20,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:35:20,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:20,932 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:35:21,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:22,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:35:22,343 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:35:22,343 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:35:22,344 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:35:22,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:23,154 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:35:23,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:24,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:25,156 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:35:25,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:35:25,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:26,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:27,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:28,286 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:35:31,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:35:33,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:34,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:35,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:35:35,343 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:35:35,343 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:35:35,345 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:35:35,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:35,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:35:35,931 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:35:36,167 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:35:37,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:35:37,168 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:35:42,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:35:47,167 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:35:48,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:49,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:50,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:35:50,047 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:35:50,047 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:35:50,048 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:35:50,194 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:35:50,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:35:51,195 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:35:53,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:35:58,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:35:58,286 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:36:01,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:36:01,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:01,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:36:02,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:03,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:36:03,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:36:03,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:36:03,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:36:03,340 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:36:03,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:04,209 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:36:05,210 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:36:08,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:36:13,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:36:18,343 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:36:23,343 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:36:23,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:23,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:36:23,506 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:36:24,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:25,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:36:25,010 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:36:25,010 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:36:25,011 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:36:25,230 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:36:25,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:26,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:27,232 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:36:27,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:28,287 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:36:28,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:36:28,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:29,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:30,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:33,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:36:38,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:36:39,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:36:39,398 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:36:39,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:40,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:41,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:36:41,143 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:36:41,143 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:36:41,145 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:36:41,246 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:36:41,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:42,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:43,248 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:36:43,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:44,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:36:44,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:49,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:36:52,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:53,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:53,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:36:53,943 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:36:53,943 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:36:53,945 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:36:54,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:54,276 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:36:54,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:36:54,397 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:36:54,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:36:55,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:55,306 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:36:56,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:57,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:58,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:36:58,289 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:36:59,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:00,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:00,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:01,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:05,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:07,511 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:08,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:09,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:37:09,219 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:37:09,219 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:37:09,220 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:37:09,321 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:37:09,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:37:09,399 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:37:09,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:10,655 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:11,323 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:37:15,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:20,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:24,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:24,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:37:24,923 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:37:25,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:26,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:26,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:37:26,364 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:37:26,365 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:37:26,366 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:37:26,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:27,354 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:37:27,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:28,290 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:37:28,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:29,356 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:37:29,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:30,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:31,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:31,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:32,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:35,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:36,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:37,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:37,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:37:37,131 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:37:37,131 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:37:37,132 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:37:37,364 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:37:37,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:38,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:39,366 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:37:39,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:39,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:37:39,921 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:37:40,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:41,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:42,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:42,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:43,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:44,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:45,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:46,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:47,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:48,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:48,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:49,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:50,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:51,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:52,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:53,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:53,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:54,552 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:54,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:37:54,921 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:37:55,552 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:56,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:37:56,191 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:37:56,191 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:37:56,193 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:37:56,382 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:37:56,552 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:57,552 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:37:58,292 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:37:59,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:37:59,385 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:38:04,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:38:07,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:08,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:09,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:38:09,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:38:09,431 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:38:09,431 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:38:09,433 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:38:09,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:38:09,922 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:38:09,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:10,395 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:38:10,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:11,396 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:38:15,197 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:38:20,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:38:20,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:21,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:22,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:38:22,143 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:38:22,144 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:38:22,145 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:38:22,406 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:38:22,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:23,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:24,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:24,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:38:24,921 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:38:25,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:38:25,409 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:38:25,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:26,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:27,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:28,293 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:38:28,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:29,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:30,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:38:30,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:31,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:32,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:35,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:38:39,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:39,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:38:39,922 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:38:40,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:40,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:38:40,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:38:40,831 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:38:40,831 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:38:40,833 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:38:41,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:41,423 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:38:42,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:43,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:43,425 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:38:44,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:45,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:46,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:46,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:38:49,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:50,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:50,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:38:50,899 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:38:50,899 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:38:50,900 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:38:51,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:38:51,434 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:38:51,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:38:53,436 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:38:56,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:38:58,296 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:39:02,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:39:03,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:39:03,102 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:39:03,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:04,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:04,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:39:04,799 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:39:04,799 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:39:04,801 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:39:05,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:05,449 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:39:06,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:07,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:07,450 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:39:07,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:39:12,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:39:17,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:39:22,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:39:27,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:39:28,296 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:39:33,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:39:34,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:39:34,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:34,936 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:39:35,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:36,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:39:36,383 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:39:36,383 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:39:36,385 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:39:36,474 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:39:36,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:37,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:38,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:39:38,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:39,476 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:39:39,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:40,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:39:43,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:39:48,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:39:53,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:39:58,298 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:39:59,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:40:04,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:40:04,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:40:04,472 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:40:04,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:05,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:06,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:40:06,099 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:40:06,099 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:40:06,102 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:40:06,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:06,519 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:40:07,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:07,520 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:40:08,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:09,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:40:09,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:10,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:11,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:12,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:13,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:14,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:15,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:15,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:40:16,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:18,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:19,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:40:19,472 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:40:19,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:20,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:40:20,323 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:40:20,323 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:40:20,325 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:40:20,547 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:40:20,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:21,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:40:21,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:22,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:23,550 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:40:23,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:24,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:25,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:26,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:40:26,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:27,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:28,300 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:40:28,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:29,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:30,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:31,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:32,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:40:32,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:33,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:34,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:40:34,472 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:40:34,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:37,764 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:40:41,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:42,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:42,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:40:43,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:40:43,311 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:40:43,311 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:40:43,313 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:40:43,582 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:40:43,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:44,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:45,584 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:40:45,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:46,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:47,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:48,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:48,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:40:49,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:40:49,472 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:40:49,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:40:54,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:40:58,301 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:41:00,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:05,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:10,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:41:10,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:41:10,040 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:41:11,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:41:11,291 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:11,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:41:11,551 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:41:11,551 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:41:11,552 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:41:11,606 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:41:12,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:41:12,607 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:41:16,553 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:21,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:26,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:28,302 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:41:32,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:37,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:42,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:44,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:41:44,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:41:44,503 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:41:45,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:41:45,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:41:45,963 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:41:45,963 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:41:45,965 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:41:46,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:41:46,654 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:41:46,654 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:41:47,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:52,966 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:57,966 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:41:58,304 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:42:03,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:42:08,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:42:13,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:42:15,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:42:15,042 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:42:15,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:16,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:16,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:42:16,467 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:42:16,467 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:42:16,469 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:42:16,680 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:42:17,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:18,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:18,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:42:18,682 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:42:19,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:20,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:21,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:22,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:23,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:24,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:24,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:42:25,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:26,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:28,307 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:42:29,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:42:34,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:42:39,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:42:40,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:42:40,267 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:42:40,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:41,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:41,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:42:41,619 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:42:41,619 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:42:41,620 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:42:41,703 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:42:42,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:42,704 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:42:43,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:44,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:44,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:42:45,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:46,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:50,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:42:55,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:42:58,307 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:42:58,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:42:58,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:42:58,931 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:42:59,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:00,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:00,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:43:00,437 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:43:00,437 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:43:00,439 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:43:00,736 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:43:00,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:01,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:02,738 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:43:02,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:05,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:10,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:15,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:18,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:43:18,350 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:43:18,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:19,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:19,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:43:19,875 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:43:19,875 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:43:19,877 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:43:20,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:20,786 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:43:20,786 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:43:20,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:21,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:22,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:23,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:24,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:25,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:26,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:26,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:27,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:28,308 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:43:28,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:29,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:30,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:31,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:36,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:41,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:46,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:46,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:46,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:43:46,906 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:43:47,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:48,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:43:48,375 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:43:48,375 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:43:48,377 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:43:48,825 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:43:48,904 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:49,904 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:50,827 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:43:50,904 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:43:51,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:56,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:43:58,311 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:44:02,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:44:06,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:06,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:44:06,790 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:44:07,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:08,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:44:08,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:44:08,271 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:44:08,271 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:44:08,272 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:44:08,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:08,843 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:44:09,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:10,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:10,845 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:44:13,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:44:18,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:44:23,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:44:23,041 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:44:23,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:23,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:44:24,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:24,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:44:24,403 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:44:24,403 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:44:24,404 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:44:24,857 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:44:25,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:26,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:26,859 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:44:27,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:28,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:28,311 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:44:28,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:44:29,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:30,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:33,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:44:38,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:44:43,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:44:43,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:43,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:44:43,438 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:44:44,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:44,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:44:44,727 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:44:44,727 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:44:44,728 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:44:44,873 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:44:45,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:46,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:46,875 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:44:47,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:48,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:44:48,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:49,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:50,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:51,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:52,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:53,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:44:53,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:44:58,312 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:44:59,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:04,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:06,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:06,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:45:06,608 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:45:07,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:08,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:45:08,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:45:08,039 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:45:08,040 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:45:08,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:08,893 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:45:08,893 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:45:09,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:10,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:10,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:11,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:12,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:15,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:20,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:25,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:25,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:25,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:45:25,985 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:45:26,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:27,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:45:27,583 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:45:27,583 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:45:27,585 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:45:27,925 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:45:27,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:28,316 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:45:28,926 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:45:31,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:36,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:37,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:38,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:39,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:45:39,431 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:45:39,431 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:45:39,432 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:45:39,952 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:45:39,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:40,952 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:45:40,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:40,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:45:40,983 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:45:41,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:42,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:42,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:43,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:44,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:45,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:46,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:47,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:47,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:48,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:49,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:50,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:51,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:52,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:52,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:55,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:55,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:45:55,983 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:45:56,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:56,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:45:56,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:45:56,815 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:45:56,816 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:45:56,968 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:45:57,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:57,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:45:58,316 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:45:58,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:45:58,971 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:45:59,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:00,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:01,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:02,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:03,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:46:08,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:46:13,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:46:18,319 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:46:18,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:46:18,690 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:46:18,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:19,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:20,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:46:20,030 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:46:20,030 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:46:20,032 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:46:20,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:21,004 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:46:21,004 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:46:24,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:46:28,317 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:46:29,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:46:31,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:32,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:32,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:46:32,515 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:46:32,515 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:46:32,517 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:46:33,015 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:46:33,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:33,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:46:33,690 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:46:34,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:34,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:46:35,017 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:46:35,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:36,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:37,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:38,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:39,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:40,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:40,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:46:41,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:42,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:43,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:44,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:45,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:46,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:46:46,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:47,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:48,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:48,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:46:48,690 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:46:48,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:46:48,983 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:46:48,984 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:46:48,985 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:46:49,029 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:46:49,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:50,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:51,031 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:46:51,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:46:51,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:52,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:53,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:46:56,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:46:58,318 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:47:00,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:01,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:01,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:47:02,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:47:02,379 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:47:02,379 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:47:02,380 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:47:02,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:03,043 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:47:05,045 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:47:07,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:47:12,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:47:17,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:47:22,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:47:25,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:25,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:47:25,849 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:47:26,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:27,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:47:27,259 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:47:27,259 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:47:27,261 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:47:27,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:28,066 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:47:28,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:47:28,322 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:47:28,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:29,066 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:47:29,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:30,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:31,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:33,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:47:38,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:47:43,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:47:48,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:47:51,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:51,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:47:51,505 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:47:52,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:53,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:47:53,067 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:47:53,067 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:47:53,069 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:47:53,124 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:47:53,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:47:54,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:47:55,126 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:47:58,321 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:47:59,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:04,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:09,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:09,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:48:09,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:09,774 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:48:10,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:11,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:48:11,239 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:48:11,239 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:48:11,241 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:48:11,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:12,141 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:48:12,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:13,142 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:48:15,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:20,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:25,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:27,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:27,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:48:27,682 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:48:28,323 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:48:28,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:29,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:48:29,087 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:48:29,087 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:48:29,088 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:48:29,206 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:48:29,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:31,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:31,208 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:48:36,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:41,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:41,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:42,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:42,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:48:42,681 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:48:42,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:48:42,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:48:42,955 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:48:42,957 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:48:43,219 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:48:43,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:44,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:45,221 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:48:45,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:46,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:46,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:48:51,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:56,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:48:58,326 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:49:02,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:49:04,468 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:49:04,469 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:49:04,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:05,467 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:06,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:49:06,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:49:06,039 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:49:06,040 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:49:06,239 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:49:06,468 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:07,240 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:49:07,468 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:08,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:49:13,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:49:18,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:49:21,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:21,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:49:21,674 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:49:22,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:23,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:49:23,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:49:23,045 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:49:23,045 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:49:23,047 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:49:23,254 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:49:23,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:24,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:25,256 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:49:28,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:49:28,326 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:49:33,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:49:34,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:35,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:36,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:49:36,043 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:49:36,043 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:49:36,045 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:49:36,267 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:49:36,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:36,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:49:36,672 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:49:37,268 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:49:37,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:38,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:38,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:49:39,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:40,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:41,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:44,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:49:49,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:49:54,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:49:57,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:57,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:49:57,649 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:49:58,327 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:49:58,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:49:59,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:49:59,419 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:49:59,419 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:49:59,420 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:49:59,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:00,303 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:50:00,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:00,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:01,304 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:50:01,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:02,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:05,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:10,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:15,651 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:20,651 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:21,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:21,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:50:21,123 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:50:22,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:22,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:50:22,767 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:50:22,767 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:50:22,769 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:50:23,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:23,338 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:50:24,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:25,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:25,355 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:50:25,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:26,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:27,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:28,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:28,331 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:50:29,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:30,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:31,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:31,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:32,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:36,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:40,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:50:40,426 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:50:40,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:41,425 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:41,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:42,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:50:42,151 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:50:42,151 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:50:42,152 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:50:42,369 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:50:42,425 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:43,370 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:50:43,425 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:44,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:50:47,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:52,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:57,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:50:58,330 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:50:59,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:50:59,528 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:50:59,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:00,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:01,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:51:01,135 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:51:01,135 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:51:01,136 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:51:01,401 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:51:01,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:02,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:03,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:51:03,404 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:51:03,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:08,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:51:13,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:51:18,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:51:19,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:19,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:51:19,012 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:51:20,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:21,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:21,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:51:21,171 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:51:21,171 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:51:21,172 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:51:21,422 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:51:22,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:23,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:23,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:51:23,423 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:51:24,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:28,332 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:51:28,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:51:33,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:51:38,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:51:38,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:51:38,362 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:51:38,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:39,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:39,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:51:39,819 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:51:39,819 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:51:39,820 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:51:40,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:51:40,438 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:51:41,439 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:51:43,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:51:48,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:51:53,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:51:58,333 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:51:59,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:04,335 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:06,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:06,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:52:06,582 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:52:07,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:08,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:52:08,419 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:52:08,419 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:52:08,420 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:52:08,494 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:52:08,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:09,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:09,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:10,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:11,497 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:52:11,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:12,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:13,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:14,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:14,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:19,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:24,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:24,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:24,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:52:24,624 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:52:25,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:26,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:52:26,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:52:26,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:52:26,368 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:52:26,509 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:52:26,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:27,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:28,336 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:52:28,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:29,511 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:52:29,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:29,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:30,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:31,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:32,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:33,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:34,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:35,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:40,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:44,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:44,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:52:44,255 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:52:45,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:45,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:52:45,732 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:45,732 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:52:45,732 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:52:45,733 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:52:46,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:46,526 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:52:47,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:47,527 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:52:48,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:49,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:50,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:51,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:51,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:52,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:53,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:54,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:55,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:56,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:57,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:52:57,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:58,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:52:58,336 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:52:59,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:52:59,254 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:52:59,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:00,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:01,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:02,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:02,552 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:53:03,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:04,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:05,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:06,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:07,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:08,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:53:08,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:09,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:10,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:11,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:12,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:13,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:14,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:53:14,254 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:53:14,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:14,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:53:15,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:19,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:53:23,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:24,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:24,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:53:24,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:53:24,663 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:53:24,663 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:53:24,665 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:53:25,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:25,580 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:53:26,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:27,581 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:53:28,337 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:53:30,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:53:35,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:53:40,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:53:43,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:43,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:53:43,684 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:53:44,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:45,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:53:45,119 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:53:45,119 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:53:45,120 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:53:45,636 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:53:45,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:46,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:53:46,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:47,638 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:53:47,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:48,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:49,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:50,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:51,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:51,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:53:52,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:53,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:54,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:55,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:56,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:57,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:53:57,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:58,340 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:53:58,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:53:58,682 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:53:58,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:53:59,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:00,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:02,945 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:54:07,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:54:12,947 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:54:17,947 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:54:22,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:22,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:54:22,461 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:54:23,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:23,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:54:23,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:54:23,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:54:23,867 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:54:23,868 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:54:24,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:24,670 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:54:25,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:25,671 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:54:26,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:27,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:28,340 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:54:28,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:29,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:54:29,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:34,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:54:39,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:54:41,732 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:41,732 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:54:41,732 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:54:42,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:43,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:54:43,295 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:54:43,295 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:54:43,296 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:54:43,685 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:54:43,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:44,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:45,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:54:45,687 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:54:45,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:46,732 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:47,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:48,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:49,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:50,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:54:50,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:54:55,732 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:54:58,342 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:55:01,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:55:01,307 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:55:01,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:01,508 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:55:02,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:02,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:55:02,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:55:02,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:55:02,721 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:55:03,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:03,701 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:55:05,702 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:55:06,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:55:11,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:55:16,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:55:17,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:17,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:55:17,771 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:55:18,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:19,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:55:19,627 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:55:19,627 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:55:19,628 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:55:19,732 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:55:19,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:20,733 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:55:20,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:21,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:22,629 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:55:22,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:23,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:24,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:25,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:26,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:27,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:55:28,343 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:55:33,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:55:38,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:55:41,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:41,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:55:41,858 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:55:42,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:43,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:55:43,344 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:55:43,344 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:55:43,346 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:55:43,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:55:43,750 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:55:43,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:44,751 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:55:44,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:45,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:46,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:47,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:48,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:48,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:55:49,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:50,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:51,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:52,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:53,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:54,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:55:54,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:55:58,345 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:56:00,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:05,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:10,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:15,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:15,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:15,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:56:15,906 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:56:16,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:17,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:56:17,443 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:56:17,443 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:56:17,445 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:56:17,793 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:56:17,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:18,794 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:56:18,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:20,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:25,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:28,346 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:56:31,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:34,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:34,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:56:34,183 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:56:35,181 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:35,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:56:35,727 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:56:35,727 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:56:35,729 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:56:35,810 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:56:36,181 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:36,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:36,811 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:56:37,181 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:38,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:39,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:42,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:47,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:52,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:54,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:56:54,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:54,361 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:56:55,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:55,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:56:55,902 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:56:55,903 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:56:55,904 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:56:56,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:56,846 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:56:56,846 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:56:57,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:57,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:56:58,347 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:56:58,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:56:59,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:00,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:01,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:03,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:57:08,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:57:13,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:57:14,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:57:14,368 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:57:14,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:15,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:15,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:57:15,791 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:57:15,791 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:57:15,792 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:57:15,862 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:57:16,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:16,863 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:57:18,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:57:23,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:57:27,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:28,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:28,366 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:57:28,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:57:28,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:57:28,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:57:28,720 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:57:28,891 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:57:29,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:29,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:57:29,368 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:57:29,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:57:30,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:30,893 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:57:31,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:32,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:33,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:35,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:57:40,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:57:45,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:57:50,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:57:53,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:57:53,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:53,711 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:57:54,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:55,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:57:55,247 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:57:55,247 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:57:55,255 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:57:55,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:55,912 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:57:56,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:57:56,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:56,913 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:57:57,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:57:58,351 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:57:58,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:01,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:58:06,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:58:08,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:08,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:58:08,843 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:58:09,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:10,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:58:10,303 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:58:10,303 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:58:10,304 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:58:10,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:10,927 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:58:11,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:12,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:58:12,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:12,929 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:58:13,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:14,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:17,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:58:22,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:58:27,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:58:28,352 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:58:29,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:58:29,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:29,638 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:58:30,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:31,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:58:31,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:58:31,003 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:58:31,005 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:58:31,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:31,946 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:58:32,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:32,947 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:58:33,005 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:58:33,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:34,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:35,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:36,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:37,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:38,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:38,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:58:39,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:40,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:41,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:42,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:43,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:44,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:58:44,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:58:44,637 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:58:44,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:45,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:46,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:47,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:48,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:49,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:58:49,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:50,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:51,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:58:51,343 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:58:51,343 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:58:51,345 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:58:51,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:51,963 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:58:52,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:52,964 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:58:53,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:54,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:58:54,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:55,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:56,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:57,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:58,355 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:58:58,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:59,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:58:59,637 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:58:59,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:58:59,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:59:00,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:01,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:02,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:03,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:04,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:05,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:59:05,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:06,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:07,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:08,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:09,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:10,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:59:10,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:11,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:12,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:13,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:14,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:59:14,637 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:59:14,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:15,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:15,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:59:16,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:17,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:18,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:19,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:20,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:21,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:59:21,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:22,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:23,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:24,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:25,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:26,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:27,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:59:27,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:28,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:59:28,191 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:59:28,191 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:59:28,193 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:59:28,355 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:59:28,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:28,998 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:59:28,999 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:59:29,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:59:29,637 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:59:29,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:30,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:32,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:59:36,424 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:37,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:37,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:59:37,843 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:59:37,843 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:59:37,845 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:59:38,008 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:59:38,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:38,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:59:39,009 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:59:39,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:40,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:43,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:59:48,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:59:53,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 06:59:56,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:56,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 06:59:56,152 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 06:59:57,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:57,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 06:59:57,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 06:59:57,635 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 06:59:57,637 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 06:59:58,058 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 06:59:58,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 06:59:58,357 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 06:59:59,059 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 06:59:59,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:04,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:09,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:14,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:16,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:00:16,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:00:16,255 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:00:17,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:00:17,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:00:17,638 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:00:17,638 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:00:17,640 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:00:18,090 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:00:18,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:00:19,090 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:00:19,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:00:19,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:24,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:28,358 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:00:30,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:35,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:40,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:43,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:00:43,900 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:00:43,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:00:44,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:00:45,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:00:45,275 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:00:45,275 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:00:45,281 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:00:45,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:00:46,115 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:00:46,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:47,116 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:00:51,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:54,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:00:55,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:00:55,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:00:55,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:00:55,635 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:00:55,652 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:00:56,125 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:00:56,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:00:56,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:00:57,126 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:00:57,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:00:58,361 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:01:02,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:01:07,363 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:01:12,363 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:01:16,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:01:16,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:01:16,805 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:01:17,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:01:18,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:01:18,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:01:18,443 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:01:18,443 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:01:18,445 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:01:18,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:01:19,180 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:01:19,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:01:20,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:01:21,182 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:01:21,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:01:23,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:01:28,361 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:01:29,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:01:34,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:01:39,363 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:01:44,363 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:01:49,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:01:49,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:01:49,299 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:01:49,520 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:01:50,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:01:50,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:01:50,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:01:50,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:01:50,797 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:01:51,211 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:01:51,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:01:52,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:01:53,213 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:01:53,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:01:54,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:01:58,362 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:02:00,364 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:05,364 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:07,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:02:07,981 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:02:08,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:08,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:09,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:02:09,531 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:02:09,531 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:02:09,533 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:02:09,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:10,229 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:02:10,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:10,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:11,230 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:02:11,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:12,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:13,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:14,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:15,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:15,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:16,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:20,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:25,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:28,365 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:02:31,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:36,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:02:36,041 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:02:36,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:37,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:37,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:37,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:02:37,539 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:02:37,539 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:02:37,540 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:02:38,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:38,254 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:02:39,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:39,254 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:02:40,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:41,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:42,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:43,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:02:43,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:48,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:53,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:58,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:02:58,365 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:03:03,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:03:04,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:04,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:03:04,312 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:03:05,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:05,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:03:05,663 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:03:05,663 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:03:05,664 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:03:06,277 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:03:06,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:07,278 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:03:07,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:08,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:08,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:03:13,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:03:16,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:17,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:18,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:03:18,059 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:03:18,059 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:03:18,061 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:03:18,289 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:03:18,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:19,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:03:19,290 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:03:24,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:03:28,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:03:29,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:03:34,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:03:38,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:38,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:03:38,551 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:03:39,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:39,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:03:40,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:03:40,015 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:03:40,015 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:03:40,030 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:03:40,308 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:03:40,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:41,309 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:03:41,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:42,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:45,031 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:03:50,031 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:03:53,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:53,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:03:53,550 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:03:54,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:54,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:03:54,557 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:03:54,557 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:03:54,559 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:03:55,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:55,349 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:03:55,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:03:56,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:57,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:57,351 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:03:58,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:03:58,370 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:03:59,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:00,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:01,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:01,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:04:06,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:04:11,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:04:14,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:14,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:04:14,532 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:04:15,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:16,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:04:16,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:04:16,175 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:04:16,176 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:04:16,397 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:04:16,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:17,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:04:17,398 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:04:17,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:18,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:19,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:20,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:21,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:22,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:04:22,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:23,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:24,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:25,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:26,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:27,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:04:28,370 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:04:33,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:04:38,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:04:43,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:04:47,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:04:47,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:47,321 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:04:48,319 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:48,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:04:48,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:04:48,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:04:48,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:04:48,721 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:04:49,320 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:49,425 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:04:50,320 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:04:51,427 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:04:53,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:04:58,372 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:04:59,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:05:03,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:05:03,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:03,913 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:05:04,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:05,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:05:05,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:05:05,583 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:05:05,583 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:05:05,585 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:05:05,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:06,441 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:05:06,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:07,442 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:05:07,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:08,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:10,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:05:15,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:05:20,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:05:23,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:05:23,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:23,943 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:05:24,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:25,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:05:25,490 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:05:25,491 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:05:25,492 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:05:25,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:26,489 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:05:26,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:05:26,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:27,490 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:05:27,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:28,373 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:05:28,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:29,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:30,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:31,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:31,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:05:32,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:33,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:34,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:35,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:36,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:37,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:37,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:05:38,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:05:38,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:38,943 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:05:39,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:40,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:41,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:42,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:43,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:05:43,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:44,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:45,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:46,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:47,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:48,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:05:48,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:49,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:50,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:51,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:52,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:05:52,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:05:52,235 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:05:52,236 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:05:52,512 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:05:52,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:53,513 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:05:53,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:05:53,943 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:05:53,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:05:54,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:05:58,376 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:05:59,377 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:04,377 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:09,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:14,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:19,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:21,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:06:21,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:21,264 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:06:22,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:22,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:06:22,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:06:22,587 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:06:22,588 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:06:23,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:23,539 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:06:24,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:24,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:25,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:25,540 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:06:26,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:27,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:28,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:28,376 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:06:29,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:30,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:30,377 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:31,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:32,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:36,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:36,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:36,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:06:36,264 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:06:37,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:37,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:06:37,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:06:37,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:06:37,812 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:06:38,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:38,552 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:06:39,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:39,553 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:06:40,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:41,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:42,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:42,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:47,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:52,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:53,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:06:53,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:53,731 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:06:54,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:55,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:06:55,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:06:55,823 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:06:55,823 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:06:55,824 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:06:56,568 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:06:57,569 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:06:57,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:06:58,377 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:07:03,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:07:08,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:07:13,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:07:18,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:07:23,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:07:24,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:07:24,479 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:07:24,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:25,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:26,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:07:26,363 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:07:26,363 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:07:26,365 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:07:26,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:26,609 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:07:27,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:28,380 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:07:28,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:07:28,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:29,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:29,612 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:07:30,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:33,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:07:38,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:07:41,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:07:41,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:41,274 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:07:42,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:43,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:07:43,081 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:07:43,082 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:07:43,083 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:07:43,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:43,625 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:07:44,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:07:44,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:45,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:45,626 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:07:46,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:47,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:48,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:07:49,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:07:54,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:07:58,381 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:07:59,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:03,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:03,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:08:03,058 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:08:04,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:04,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:04,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:08:04,823 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:08:04,823 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:08:04,825 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:08:05,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:05,657 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:08:06,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:07,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:07,659 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:08:08,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:09,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:10,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:10,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:11,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:12,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:13,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:14,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:15,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:16,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:16,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:17,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:18,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:08:18,057 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:08:18,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:19,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:20,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:21,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:21,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:24,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:25,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:25,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:08:25,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:08:25,559 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:08:25,560 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:08:25,688 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:08:26,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:26,561 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:27,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:27,690 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:08:28,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:28,383 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:08:32,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:37,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:42,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:47,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:49,685 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:08:49,685 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:08:49,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:50,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:51,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:08:51,223 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:08:51,223 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:08:51,225 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:08:51,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:08:51,709 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:08:53,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:53,711 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:08:58,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:08:58,384 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:09:03,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:09:06,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:06,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:09:06,110 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:09:07,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:07,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:09:07,741 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:09:07,741 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:09:07,742 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:09:08,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:08,723 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:09:08,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:09:09,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:09,723 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:09:13,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:09:16,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:17,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:17,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:09:17,735 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:09:17,735 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:09:17,737 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:09:17,738 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:09:18,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:19,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:19,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:09:19,740 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:09:24,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:09:28,385 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:09:29,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:29,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:09:29,738 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:09:29,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:09:30,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:31,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:09:31,387 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:09:31,387 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:09:31,388 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:09:31,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:31,766 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:09:32,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:33,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:33,768 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:09:34,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:35,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:09:35,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:36,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:37,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:38,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:39,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:40,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:09:40,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:41,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:42,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:43,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:44,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:09:44,738 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:09:44,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:45,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:09:45,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:09:45,175 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:09:45,176 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:09:45,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:45,779 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:09:46,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:09:46,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:47,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:47,781 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:09:48,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:49,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:50,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:51,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:09:56,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:57,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:57,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:09:58,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:09:58,111 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:09:58,111 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:09:58,112 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:09:58,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:58,387 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:09:58,808 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:09:59,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:09:59,809 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:10:02,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:10:07,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:10:12,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:10:14,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:10:14,059 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:10:14,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:15,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:15,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:10:15,527 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:10:15,527 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:10:15,544 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:10:15,830 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:10:16,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:16,831 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:10:17,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:17,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:10:18,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:19,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:20,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:23,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:10:25,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:26,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:27,216 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:10:27,217 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:10:27,217 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:10:27,218 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:10:27,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:27,842 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:10:28,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:10:28,390 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:10:28,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:28,843 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:10:33,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:10:37,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:10:37,368 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:10:37,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:38,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:38,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:10:38,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:10:38,831 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:10:38,831 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:10:38,832 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:10:38,867 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:10:39,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:40,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:40,869 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:10:41,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:42,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:43,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:44,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:10:44,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:45,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:10:49,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:10:54,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:10:58,393 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:10:59,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:00,135 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:00,135 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:11:00,135 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:11:01,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:01,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:11:01,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:11:01,587 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:11:01,589 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:11:01,902 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:11:02,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:02,903 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:11:03,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:04,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:04,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:05,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:10,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:13,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:14,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:14,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:11:14,887 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:11:14,887 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:11:14,888 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:11:14,915 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:11:15,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:11:15,134 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:11:15,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:15,481 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:16,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:16,917 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:11:17,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:18,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:19,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:20,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:20,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:21,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:22,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:23,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:24,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:25,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:26,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:26,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:28,393 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:11:32,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:34,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:11:34,261 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:11:34,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:35,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:35,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:11:35,703 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:11:35,703 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:11:35,705 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:11:35,936 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:11:36,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:36,937 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:11:37,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:37,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:38,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:11:42,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:47,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:52,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:57,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:11:58,395 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:12:00,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:12:00,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:00,654 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:12:01,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:02,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:12:02,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:12:02,039 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:12:02,041 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:12:02,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:02,962 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:12:02,963 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:12:03,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:12:03,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:08,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:12:13,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:12:18,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:12:23,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:12:23,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:12:23,200 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:12:23,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:24,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:24,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:12:24,563 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:12:24,563 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:12:24,565 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:12:24,980 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:12:25,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:26,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:26,982 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:12:28,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:12:28,400 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:12:33,400 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:12:38,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:12:43,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:12:48,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:12:53,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:12:56,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:56,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:12:56,330 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:12:57,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:57,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:12:57,787 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:12:57,787 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:12:57,789 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:12:58,022 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:12:58,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:58,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:12:59,023 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:12:59,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:12:59,400 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:00,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:01,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:05,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:10,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:15,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:17,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:17,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:13:17,215 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:13:18,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:18,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:13:18,743 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:13:18,743 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:13:18,745 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:13:19,040 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:13:19,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:20,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:20,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:21,042 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:13:21,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:22,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:23,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:24,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:25,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:26,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:28,400 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:13:30,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:31,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:31,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:31,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:13:31,987 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:13:31,987 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:13:32,002 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:13:32,053 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:13:32,213 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:13:32,214 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:13:32,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:33,054 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:13:33,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:34,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:35,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:36,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:37,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:37,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:38,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:39,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:40,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:41,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:42,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:42,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:43,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:44,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:47,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:52,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:13:52,421 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:13:52,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:52,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:53,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:53,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:13:53,859 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:13:53,859 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:13:53,861 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:13:54,071 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:13:54,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:55,072 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:13:55,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:56,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:57,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:58,404 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:13:58,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:13:58,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:13:59,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:03,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:14:04,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:05,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:05,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:14:05,555 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:14:05,555 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:14:05,557 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:14:06,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:06,083 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:14:07,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:07,084 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:14:07,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:14:07,420 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:14:08,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:14:13,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:14:18,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:14:19,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:20,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:20,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:14:20,639 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:14:20,639 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:14:20,641 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:14:21,098 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:14:21,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:22,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:22,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:14:22,420 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:14:23,100 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:14:23,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:23,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:14:24,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:28,404 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:14:29,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:14:34,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:14:38,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:38,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:14:38,912 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:14:39,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:40,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:14:40,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:14:40,431 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:14:40,431 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:14:40,446 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:14:40,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:14:41,118 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:14:43,120 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:14:45,447 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:14:50,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:14:55,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:14:58,405 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:15:01,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:06,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:15:06,138 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:15:06,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:07,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:07,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:07,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:15:07,555 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:15:07,555 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:15:07,556 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:15:08,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:08,156 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:15:09,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:09,157 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:15:10,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:11,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:13,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:18,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:23,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:28,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:28,407 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:15:28,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:15:28,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:28,883 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:15:29,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:30,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:15:30,379 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:15:30,379 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:15:30,381 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:15:30,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:31,176 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:15:31,176 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:15:31,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:32,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:33,178 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:15:33,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:38,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:43,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:48,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:49,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:49,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:15:49,494 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:15:50,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:51,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:15:51,067 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:15:51,067 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:15:51,068 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:15:51,203 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:15:51,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:52,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:53,205 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:15:53,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:54,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:54,493 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:58,410 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:15:58,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:15:59,411 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:15:59,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:00,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:16:00,083 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:16:00,083 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:16:00,085 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:16:00,226 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:16:00,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:01,227 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:16:05,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:16:10,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:16:11,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:16:11,501 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:16:11,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:12,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:12,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:16:12,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:16:12,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:16:12,929 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:16:13,239 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:16:13,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:14,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:15,242 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:16:15,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:15,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:16:16,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:17,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:18,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:19,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:21,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:16:24,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:25,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:25,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:16:25,583 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:16:25,583 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:16:25,608 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:16:26,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:26,254 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:16:26,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:16:26,500 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:16:26,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:16:27,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:27,254 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:16:28,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:28,410 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:16:29,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:30,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:32,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:16:37,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:16:42,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:16:44,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:16:44,084 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:16:44,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:45,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:45,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:16:45,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:16:45,483 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:16:45,485 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:16:46,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:46,287 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:16:47,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:47,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:16:47,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:16:48,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:49,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:50,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:53,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:16:54,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:55,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:55,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:16:55,895 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:16:55,896 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:16:55,897 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:16:56,326 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:16:56,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:57,327 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:16:57,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:58,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:16:58,411 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:16:58,412 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:16:59,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:16:59,084 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:16:59,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:00,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:01,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:02,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:03,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:04,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:17:09,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:17:11,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:12,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:12,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:17:12,855 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:17:12,855 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:17:12,870 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:17:13,355 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:17:13,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:14,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:17:14,085 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:17:14,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:15,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:17:15,358 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:17:20,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:17:25,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:17:26,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:27,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:27,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:17:27,911 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:17:27,911 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:17:27,913 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:17:28,400 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:17:28,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:17:28,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:29,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:17:29,085 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:17:29,401 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:17:29,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:30,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:31,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:17:31,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:36,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:17:41,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:17:45,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:45,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:17:45,671 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:17:46,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:46,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:17:47,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:17:47,451 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:17:47,451 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:17:47,452 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:17:47,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:48,416 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:17:48,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:49,417 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:17:49,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:50,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:51,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:52,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:52,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:17:53,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:54,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:55,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:56,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:17:57,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:17:58,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:18:03,416 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:18:08,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:18:13,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:18:18,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:18:23,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:18:24,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:24,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:18:24,592 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:18:25,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:26,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:18:26,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:18:26,503 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:18:26,505 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:18:26,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:27,462 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:18:27,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:28,416 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:18:28,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:29,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:18:29,464 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:18:29,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:30,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:31,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:32,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:34,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:18:39,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:18:43,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:43,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:18:43,055 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:18:44,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:44,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:18:44,547 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:18:44,548 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:18:44,550 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:18:45,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:45,477 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:18:45,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:18:46,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:47,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:47,479 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:18:48,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:49,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:51,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:18:54,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:55,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:56,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:18:56,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:18:56,883 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:18:56,884 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:18:56,886 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:18:56,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:57,489 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:18:57,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:58,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:18:58,053 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:18:58,419 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:18:58,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:18:59,491 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:18:59,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:00,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:01,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:01,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:19:02,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:03,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:04,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:05,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:06,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:06,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:19:07,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:08,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:09,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:10,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:11,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:12,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:19:12,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:13,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:19:13,053 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:19:13,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:14,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:15,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:16,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:19:16,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:19:16,559 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:19:16,561 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:19:16,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:17,521 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:19:17,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:18,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:19:18,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:19,523 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:19:19,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:20,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:21,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:22,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:23,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:19:23,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:24,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:25,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:26,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:28,419 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:19:29,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:19:34,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:19:38,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:19:38,004 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:19:38,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:39,003 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:39,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:19:39,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:19:39,663 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:19:39,663 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:19:39,665 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:19:40,003 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:40,554 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:19:41,003 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:19:41,555 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:19:44,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:19:49,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:19:54,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:19:58,421 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:20:00,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:05,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:07,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:20:07,920 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:20:07,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:08,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:09,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:20:09,593 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:20:09,594 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:20:09,595 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:20:09,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:10,578 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:20:10,596 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:10,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:11,579 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:20:11,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:12,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:13,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:14,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:15,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:20,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:25,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:28,422 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:20:28,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:20:28,886 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:20:28,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:29,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:30,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:20:30,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:20:30,611 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:20:30,613 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:20:30,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:31,609 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:20:31,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:31,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:32,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:33,611 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:20:33,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:34,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:35,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:36,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:36,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:37,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:38,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:39,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:40,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:41,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:42,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:42,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:43,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:20:43,885 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:20:43,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:44,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:46,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:46,027 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:20:46,028 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:20:46,028 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:20:46,029 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:20:46,625 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:20:46,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:47,626 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:20:48,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:53,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:55,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:56,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:57,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:57,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:20:57,859 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:20:57,859 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:20:57,861 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:20:58,426 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:20:58,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:20:58,636 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:20:58,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:20:58,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:20:58,885 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:20:59,637 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:20:59,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:00,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:01,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:02,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:03,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:21:08,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:21:13,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:21:18,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:21:19,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:19,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:21:19,827 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:21:20,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:21,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:21:21,267 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:21:21,267 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:21:21,269 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:21:21,654 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:21:21,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:22,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:23,656 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:21:23,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:24,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:21:28,425 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:21:29,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:21:34,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:21:39,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:21:40,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:40,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:21:40,742 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:21:41,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:42,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:21:42,435 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:21:42,435 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:21:42,437 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:21:42,685 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:21:42,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:43,686 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:21:43,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:44,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:21:44,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:45,702 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:21:45,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:46,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:47,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:48,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:49,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:21:49,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:50,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:51,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:52,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:53,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:54,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:55,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:21:55,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:21:55,740 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:21:55,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:56,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:57,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:58,427 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:21:58,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:21:59,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:01,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:22:02,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:03,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:04,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:22:04,055 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:22:04,055 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:22:04,057 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:22:04,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:04,717 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:22:05,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:05,718 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:22:06,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:07,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:22:07,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:08,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:09,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:10,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:10,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:22:10,740 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:22:11,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:12,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:22:12,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:13,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:14,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:15,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:16,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:17,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:18,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:22:18,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:19,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:22,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:23,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:23,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:22:23,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:22:23,779 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:22:23,779 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:22:23,782 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:22:24,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:24,750 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:22:25,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:25,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:22:25,741 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:22:25,750 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:22:26,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:27,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:28,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:28,430 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:22:29,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:29,431 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:22:34,431 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:22:35,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:36,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:37,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:22:37,139 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:22:37,139 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:22:37,141 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:22:37,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:37,762 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:22:38,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:39,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:39,764 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:22:40,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:22:40,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:40,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:22:40,741 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:22:41,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:42,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:43,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:44,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:45,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:22:45,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:46,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:47,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:48,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:50,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:22:54,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:55,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:55,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:22:55,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:22:55,741 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:22:56,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:22:56,187 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:22:56,187 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:22:56,190 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:22:56,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:56,811 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:22:57,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:57,811 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:22:58,430 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:22:58,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:22:59,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:00,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:01,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:01,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:23:02,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:06,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:23:11,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:23:12,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:23:12,716 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:23:12,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:13,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:14,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:23:14,015 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:23:14,016 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:23:14,017 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:23:14,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:14,828 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:23:15,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:15,829 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:23:16,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:17,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:23:17,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:18,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:19,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:20,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:21,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:22,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:23:22,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:23,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:24,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:25,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:26,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:27,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:23:27,714 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:23:27,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:27,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:23:28,431 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:23:28,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:29,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:30,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:31,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:32,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:33,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:23:35,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:36,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:37,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:23:37,263 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:23:37,263 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:23:37,265 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:23:37,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:37,880 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:23:38,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:38,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:23:39,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:39,881 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:23:40,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:41,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:42,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:42,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:23:42,714 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:23:43,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:43,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:23:44,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:45,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:46,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:47,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:48,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:49,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:49,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:23:50,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:51,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:52,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:53,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:54,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:54,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:23:55,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:56,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:57,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:23:57,031 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:23:57,031 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:23:57,033 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:23:57,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:57,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:23:57,715 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:23:57,903 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:23:58,434 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:23:58,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:59,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:23:59,905 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:24:00,435 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:24:00,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:01,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:02,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:03,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:04,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:05,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:05,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:24:06,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:07,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:11,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:24:16,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:24:17,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:24:17,072 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:24:17,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:18,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:18,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:24:18,487 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:24:18,487 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:24:18,488 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:24:18,921 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:24:19,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:20,923 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:24:22,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:24:27,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:24:28,434 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:24:33,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:24:35,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:35,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:24:35,533 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:24:36,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:36,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:24:36,971 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:24:36,971 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:24:36,973 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:24:37,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:37,939 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:24:38,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:38,940 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:24:38,973 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:24:39,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:40,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:41,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:42,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:43,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:44,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:24:46,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:47,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:47,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:24:47,835 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:24:47,835 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:24:47,837 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:24:47,963 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:24:48,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:48,964 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:24:49,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:49,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:24:50,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:50,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:24:50,532 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:24:51,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:52,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:53,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:54,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:55,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:55,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:24:56,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:57,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:58,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:24:58,437 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:24:59,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:00,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:00,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:01,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:02,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:03,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:04,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:05,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:05,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:25:05,532 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:25:05,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:06,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:07,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:08,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:25:08,071 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:25:08,071 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:25:08,073 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:25:08,135 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:08,996 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:25:08,996 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:25:09,135 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:11,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:16,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:21,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:26,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:27,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:27,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:25:27,186 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:25:28,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:28,438 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:25:28,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:25:28,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:25:28,623 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:25:28,625 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:25:29,030 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:25:29,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:30,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:31,032 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:25:31,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:31,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:32,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:25:37,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:42,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:47,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:52,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:57,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:25:58,441 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:26:00,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:26:00,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:26:00,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:01,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:02,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:26:02,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:26:02,355 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:26:02,355 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:26:02,357 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:26:02,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:03,072 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:26:03,072 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:26:03,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:04,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:05,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:06,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:07,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:26:12,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:26:17,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:26:22,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:26:27,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:26:28,442 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:26:30,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:26:30,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:30,200 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:26:31,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:31,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:26:31,751 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:26:31,751 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:26:31,753 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:26:32,095 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:26:32,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:33,096 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:26:33,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:33,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:26:34,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:35,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:36,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:37,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:38,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:39,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:26:41,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:42,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:42,411 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:26:42,412 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:26:42,412 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:26:42,413 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:26:43,107 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:26:43,108 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:26:43,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:44,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:44,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:26:49,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:26:54,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:26:54,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:54,194 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:26:54,424 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:26:55,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:55,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:26:55,615 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:26:55,615 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:26:55,617 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:26:56,120 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:26:56,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:57,121 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:26:57,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:26:58,445 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:26:59,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:04,447 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:09,447 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:14,447 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:18,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:27:18,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:18,861 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:27:19,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:20,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:20,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:27:20,283 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:27:20,283 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:27:20,285 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:27:20,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:21,164 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:27:21,164 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:27:21,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:22,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:23,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:24,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:25,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:25,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:28,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:28,448 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:27:29,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:29,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:27:29,671 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:27:29,671 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:27:29,673 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:27:30,210 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:27:30,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:31,211 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:27:31,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:31,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:32,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:33,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:33,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:27:33,860 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:27:34,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:35,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:37,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:42,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:47,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:48,456 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:48,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:27:48,860 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:27:49,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:49,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:27:49,856 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:27:49,856 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:27:49,858 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:27:50,229 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:27:50,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:27:51,230 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:27:52,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:57,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:27:58,448 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:28:03,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:28:08,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:28:13,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:28:15,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:28:15,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:15,870 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:28:16,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:17,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:28:17,463 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:28:17,463 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:28:17,465 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:28:17,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:18,301 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:28:18,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:28:18,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:19,302 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:28:19,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:20,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:21,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:22,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:23,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:23,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:28:24,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:25,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:28,450 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:28:29,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:28:33,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:28:33,487 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:28:33,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:34,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:34,764 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:28:34,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:28:34,823 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:28:34,823 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:28:34,825 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:28:35,316 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:28:35,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:36,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:37,318 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:28:37,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:38,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:40,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:28:45,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:28:50,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:28:52,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:52,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:28:52,696 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:28:53,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:54,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:28:54,103 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:28:54,103 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:28:54,105 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:28:54,345 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:28:54,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:55,346 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:28:55,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:56,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:28:56,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:57,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:58,454 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:28:58,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:28:59,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:00,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:01,456 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:01,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:06,456 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:08,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:08,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:29:08,571 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:29:09,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:09,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:29:09,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:29:09,867 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:29:09,868 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:29:10,374 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:29:10,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:11,374 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:29:11,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:16,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:21,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:26,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:28,455 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:29:31,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:31,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:29:31,754 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:29:32,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:32,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:33,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:29:33,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:29:33,175 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:29:33,177 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:29:33,396 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:29:33,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:34,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:35,398 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:29:35,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:36,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:37,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:37,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:42,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:47,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:52,755 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:55,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:55,876 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:29:55,876 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:29:56,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:57,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:29:57,335 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:29:57,335 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:29:57,336 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:29:57,414 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:29:57,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:58,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:29:58,457 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:29:58,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:29:59,416 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:29:59,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:00,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:01,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:02,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:03,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:30:03,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:08,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:30:12,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:12,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:30:12,303 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:30:13,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:13,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:30:13,771 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:30:13,771 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:30:13,774 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:30:14,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:14,430 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:30:14,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:30:15,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:15,431 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:30:16,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:19,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:30:24,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:30:25,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:26,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:27,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:30:27,302 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:30:27,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:30:27,568 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:30:27,569 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:30:27,570 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:30:27,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:28,443 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:30:28,458 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:30:28,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:29,444 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:30:29,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:30,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:30:30,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:31,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:30:35,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:30:40,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:30:45,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:30:50,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:30:55,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:30:58,461 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:31:01,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:31:04,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:04,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:31:04,922 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:31:05,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:06,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:31:06,439 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:31:06,439 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:31:06,441 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:31:06,492 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:31:06,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:07,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:31:07,492 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:31:07,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:08,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:09,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:10,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:11,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:12,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:12,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:31:14,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:15,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:16,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:31:16,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:31:16,039 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:31:16,041 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:31:16,502 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:31:16,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:17,503 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:31:18,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:31:23,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:31:28,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:31:28,461 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:31:33,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:31:38,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:31:41,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:41,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:31:41,779 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:31:42,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:43,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:31:43,219 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:31:43,219 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:31:43,221 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:31:43,574 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:31:43,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:44,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:31:44,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:45,576 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:31:45,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:31:49,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:31:54,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:31:58,463 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:31:59,463 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:00,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:32:00,982 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:32:01,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:01,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:02,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:32:02,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:32:02,720 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:32:02,721 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:32:02,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:03,605 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:32:03,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:04,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:04,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:05,622 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:32:05,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:06,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:09,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:14,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:19,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:19,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:32:19,395 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:32:20,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:20,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:20,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:32:20,891 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:32:20,891 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:32:20,907 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:32:21,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:21,639 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:32:22,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:23,640 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:32:25,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:28,467 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:32:31,468 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:35,812 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:35,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:32:35,813 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:32:36,812 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:37,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:37,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:32:37,179 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:32:37,179 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:32:37,181 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:32:37,654 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:32:37,812 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:39,656 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:32:42,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:47,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:48,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:49,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:49,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:32:49,707 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:32:49,707 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:32:49,711 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:32:50,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:32:50,667 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:32:51,669 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:32:52,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:57,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:32:58,467 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:33:03,468 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:33:08,468 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:33:13,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:33:17,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:33:17,721 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:33:17,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:18,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:18,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:33:19,280 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:33:19,280 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:33:19,281 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:33:19,283 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:33:19,707 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:33:19,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:20,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:21,708 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:33:21,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:24,283 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:33:28,469 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:33:29,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:33:34,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:33:39,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:33:41,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:41,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:33:41,170 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:33:42,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:42,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:33:42,527 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:33:42,527 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:33:42,529 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:33:42,727 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:33:43,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:43,728 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:33:44,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:44,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:33:45,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:45,730 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:33:46,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:47,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:50,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:33:55,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:33:55,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:56,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:33:56,169 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:33:56,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:57,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:33:57,291 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:33:57,291 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:33:57,292 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:33:57,756 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:33:57,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:58,473 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:33:58,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:33:59,757 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:33:59,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:00,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:00,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:01,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:02,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:03,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:04,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:05,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:10,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:13,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:13,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:34:13,803 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:34:14,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:15,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:34:15,443 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:34:15,443 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:34:15,473 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:34:15,770 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:34:15,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:16,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:16,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:17,772 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:34:17,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:21,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:26,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:28,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:28,473 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:34:28,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:34:28,802 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:34:29,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:29,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:34:29,739 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:34:29,740 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:34:29,741 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:34:29,783 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:34:30,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:31,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:31,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:31,785 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:34:32,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:33,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:34,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:35,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:36,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:37,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:37,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:40,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:41,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:42,364 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:42,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:34:42,407 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:34:42,407 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:34:42,409 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:34:42,795 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:34:42,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:43,796 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:34:43,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:34:43,803 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:34:43,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:44,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:34:48,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:53,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:58,003 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:34:58,474 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:35:03,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:35:05,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:35:05,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:05,083 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:35:06,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:06,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:35:06,807 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:35:06,807 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:35:06,809 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:35:06,815 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:35:07,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:08,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:08,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:35:09,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:09,818 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:35:10,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:14,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:35:19,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:35:24,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:35:28,476 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:35:29,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:35:33,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:33,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:35:33,419 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:35:34,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:35,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:35:35,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:35:35,187 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:35:35,187 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:35:35,189 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:35:35,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:35,854 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:35:36,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:37,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:37,856 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:35:38,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:39,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:40,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:40,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:35:41,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:44,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:45,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:45,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:35:46,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:35:46,647 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:35:46,647 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:35:46,648 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:35:46,866 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:35:46,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:47,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:48,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:35:48,418 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:35:48,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:49,869 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:35:49,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:51,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:35:53,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:54,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:55,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:35:55,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:35:55,483 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:35:55,485 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:35:55,875 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:35:55,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:56,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:57,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:35:57,877 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:35:57,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:35:58,479 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:36:03,481 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:36:06,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:06,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:36:06,073 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:36:07,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:07,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:36:07,754 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:36:07,754 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:36:07,755 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:36:07,886 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:36:08,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:08,756 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:36:09,888 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:36:13,756 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:36:18,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:36:23,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:36:28,479 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:36:29,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:36:32,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:32,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:36:32,682 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:36:33,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:34,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:36:34,307 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:36:34,307 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:36:34,309 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:36:34,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:34,927 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:36:35,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:36:35,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:35,928 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:36:36,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:37,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:40,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:36:45,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:36:50,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:36:51,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:51,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:36:51,637 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:36:52,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:53,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:36:53,151 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:36:53,151 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:36:53,153 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:36:53,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:36:53,943 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:36:55,945 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:36:56,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:36:58,480 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:37:01,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:37:06,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:37:07,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:37:07,777 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:37:07,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:08,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:09,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:37:09,291 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:37:09,291 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:37:09,293 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:37:09,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:09,957 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:37:10,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:11,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:11,959 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:37:12,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:37:12,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:13,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:14,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:15,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:16,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:17,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:37:17,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:18,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:19,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:20,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:21,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:22,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:37:22,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:37:22,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:22,777 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:37:23,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:24,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:25,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:26,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:27,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:28,484 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:37:28,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:37:28,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:29,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:37:29,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:37:29,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:37:29,267 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:37:29,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:29,977 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:37:30,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:31,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:31,978 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:37:34,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:37:38,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:37:38,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:38,505 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:37:39,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:39,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:37:40,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:37:40,155 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:37:40,155 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:37:40,156 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:37:40,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:40,988 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:37:41,989 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:37:45,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:37:50,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:37:54,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:37:54,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:54,010 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:37:55,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:55,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:37:55,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:37:55,431 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:37:55,431 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:37:55,433 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:37:56,007 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:37:56,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:57,008 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:37:57,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:58,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:37:58,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:38:00,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:38:05,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:38:10,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:38:15,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:38:19,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:19,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:38:19,090 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:38:20,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:20,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:38:20,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:38:20,563 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:38:20,563 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:38:20,566 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:38:21,027 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:38:21,027 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:38:21,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:22,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:23,029 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:38:23,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:24,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:25,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:26,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:26,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:38:27,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:28,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:28,489 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:38:29,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:30,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:31,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:32,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:38:32,091 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:33,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:34,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:38:34,089 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:38:34,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:35,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:36,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:37,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:37,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:38:38,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:39,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:40,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:41,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:42,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:43,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:38:43,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:44,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:45,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:45,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:38:45,975 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:38:45,975 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:38:45,978 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:38:46,067 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:38:46,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:47,068 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:38:47,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:48,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:48,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:38:49,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:38:49,089 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:38:49,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:50,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:51,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:52,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:53,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:54,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:38:54,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:55,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:56,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:57,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:58,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:38:58,139 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:38:58,139 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:38:58,141 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:38:58,492 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:38:58,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:38:59,079 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:38:59,079 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:39:00,493 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:05,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:10,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:15,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:19,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:19,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:39:19,694 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:39:20,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:20,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:21,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:39:21,249 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:39:21,249 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:39:21,251 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:39:21,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:22,115 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:39:22,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:23,116 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:39:23,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:24,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:25,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:26,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:26,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:28,491 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:39:32,493 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:33,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:34,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:34,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:39:34,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:39:34,747 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:39:34,983 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:39:34,983 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:39:34,985 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:39:35,127 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:39:35,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:36,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:37,129 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:39:37,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:37,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:42,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:47,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:52,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:57,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:39:58,493 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:39:58,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:39:58,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:39:58,803 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:39:59,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:00,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:40:00,383 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:40:00,383 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:40:00,385 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:40:00,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:01,152 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:40:01,152 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:40:03,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:40:08,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:40:13,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:40:18,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:40:23,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:40:28,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:40:28,494 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:40:28,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:28,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:40:28,721 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:40:29,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:30,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:40:30,135 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:40:30,135 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:40:30,137 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:40:30,218 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:40:30,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:31,219 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:40:31,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:34,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:40:39,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:40:39,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:40,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:40,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:40:40,819 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:40:40,819 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:40:40,821 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:40:41,230 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:40:41,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:42,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:43,231 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:40:43,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:43,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:40:43,720 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:40:44,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:40:44,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:40:49,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:40:54,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:40:58,497 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:40:59,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:04,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:09,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:14,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:18,116 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:18,116 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:41:18,117 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:41:19,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:19,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:19,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:41:19,567 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:41:19,567 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:41:19,569 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:41:20,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:20,291 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:41:21,117 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:21,292 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:41:22,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:23,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:24,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:25,117 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:28,497 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:41:30,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:35,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:37,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:37,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:41:37,393 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:41:38,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:38,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:41:38,823 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:41:38,823 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:41:38,824 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:41:39,308 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:41:39,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:40,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:40,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:41,310 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:41:45,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:50,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:52,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:41:52,838 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:41:52,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:53,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:54,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:41:54,327 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:41:54,327 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:41:54,328 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:41:54,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:55,324 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:41:55,325 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:41:55,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:56,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:41:56,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:57,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:58,499 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:41:58,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:41:59,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:01,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:42:04,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:05,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:05,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:42:05,551 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:42:05,551 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:42:05,553 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:42:06,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:06,351 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:42:06,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:42:07,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:07,352 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:42:07,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:42:07,837 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:42:08,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:09,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:10,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:11,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:12,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:12,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:42:13,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:14,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:15,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:16,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:17,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:17,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:42:18,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:19,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:20,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:21,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:22,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:22,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:42:22,837 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:42:23,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:42:23,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:24,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:25,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:26,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:27,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:28,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:42:28,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:28,502 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:42:29,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:42:29,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:29,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:42:29,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:42:29,369 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:42:29,372 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:42:30,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:31,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:31,374 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:42:32,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:33,324 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:33,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:42:34,324 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:35,324 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:36,324 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:37,324 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:37,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:42:37,839 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:42:38,324 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:39,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:42:39,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:44,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:42:49,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:42:50,319 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:51,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:51,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:42:51,722 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:42:51,722 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:42:51,724 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:42:52,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:42:52,392 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:42:52,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:42:52,839 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:42:53,393 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:42:55,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:42:58,501 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:43:00,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:43:05,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:43:09,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:09,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:43:09,407 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:43:10,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:10,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:43:10,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:43:10,799 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:43:10,799 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:43:10,801 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:43:11,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:11,437 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:43:13,439 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:43:15,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:43:18,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:19,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:20,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:43:20,111 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:43:20,111 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:43:20,113 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:43:20,475 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:43:20,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:21,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:43:21,476 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:43:21,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:22,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:23,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:24,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:43:24,407 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:43:24,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:25,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:26,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:26,655 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:43:27,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:28,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:43:32,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:43:37,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:43:42,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:43:43,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:43:43,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:43,670 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:43:44,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:45,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:43:45,063 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:43:45,063 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:43:45,064 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:43:45,559 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:43:45,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:46,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:47,561 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:43:47,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:48,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:43:48,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:49,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:50,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:51,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:52,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:53,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:43:53,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:54,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:43:58,505 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:43:59,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:03,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:03,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:44:03,859 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:44:04,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:05,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:05,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:44:05,391 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:44:05,391 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:44:05,393 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:44:05,607 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:44:05,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:06,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:07,609 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:44:07,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:08,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:09,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:10,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:15,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:20,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:21,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:44:21,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:21,740 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:44:22,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:23,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:44:23,487 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:44:23,487 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:44:23,489 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:44:23,623 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:44:23,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:24,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:25,625 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:44:25,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:26,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:26,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:28,505 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:44:31,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:34,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:35,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:36,728 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:36,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:44:36,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:44:36,771 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:44:36,771 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:44:36,773 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:44:36,773 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:44:36,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:37,638 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:44:37,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:38,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:39,639 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:44:39,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:40,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:41,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:41,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:42,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:43,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:44,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:46,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:51,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:56,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:44:56,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:56,887 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:44:57,188 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:44:57,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:58,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:44:58,347 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:44:58,347 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:44:58,348 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:44:58,507 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:44:58,690 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:44:58,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:44:59,691 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:44:59,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:00,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:01,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:02,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:45:02,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:03,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:04,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:05,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:06,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:07,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:08,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:45:08,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:09,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:13,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:45:15,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:15,411 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:45:15,411 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:45:16,409 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:16,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:45:16,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:45:16,815 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:45:16,817 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:45:17,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:17,721 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:45:18,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:19,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:19,723 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:45:19,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:45:20,411 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:21,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:25,411 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:45:28,508 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:45:30,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:45:35,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:45:36,335 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:36,335 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:45:36,336 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:45:37,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:37,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:45:37,727 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:45:37,727 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:45:37,728 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:45:37,737 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:45:38,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:39,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:39,739 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:45:40,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:45:45,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:45:50,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:45:53,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:53,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:45:53,268 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:45:54,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:54,948 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:45:54,949 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:45:54,949 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:45:54,950 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:45:55,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:55,753 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:45:55,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:45:56,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:57,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:57,755 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:45:58,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:45:58,511 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:45:59,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:00,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:01,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:01,512 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:46:02,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:03,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:04,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:05,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:06,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:07,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:46:07,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:08,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:46:08,107 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:46:08,107 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:46:08,108 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:46:08,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:46:08,267 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:46:08,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:08,765 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:46:09,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:09,766 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:46:10,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:11,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:12,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:46:17,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:46:22,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:46:27,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:46:28,510 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:46:32,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:46:32,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:32,098 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:46:33,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:33,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:46:33,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:46:33,783 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:46:33,783 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:46:33,784 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:46:33,787 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:46:34,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:35,789 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:46:38,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:46:41,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:42,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:43,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:46:43,211 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:46:43,211 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:46:43,213 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:46:43,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:46:43,798 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:46:44,213 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:46:45,814 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:46:49,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:46:54,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:46:58,512 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:46:59,513 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:04,513 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:09,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:09,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:09,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:47:09,975 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:47:10,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:11,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:47:11,387 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:47:11,387 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:47:11,389 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:47:11,852 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:47:11,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:12,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:13,854 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:47:13,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:14,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:14,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:15,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:16,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:17,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:18,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:19,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:19,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:20,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:21,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:22,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:23,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:24,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:47:24,974 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:47:25,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:25,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:25,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:26,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:27,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:28,515 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:47:28,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:29,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:30,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:30,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:32,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:32,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:47:32,043 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:47:32,043 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:47:32,045 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:47:32,892 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:47:33,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:33,893 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:47:36,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:41,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:46,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:47,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:47:47,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:47,625 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:47:48,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:49,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:47:49,123 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:47:49,123 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:47:49,124 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:47:49,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:49,909 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:47:50,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:51,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:51,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:51,911 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:47:52,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:53,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:54,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:55,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:56,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:47:56,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:57,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:58,515 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:47:58,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:47:59,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:00,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:01,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:02,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:48:02,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:48:02,623 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:48:02,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:03,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:04,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:07,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:48:09,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:10,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:11,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:48:11,711 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:48:11,711 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:48:11,713 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:48:11,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:11,960 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:48:12,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:13,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:48:13,961 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:48:18,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:48:23,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:48:28,516 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:48:28,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:48:28,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:28,570 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:48:28,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:48:29,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:30,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:48:30,107 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:48:30,107 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:48:30,109 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:48:30,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:30,991 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:48:31,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:31,992 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:48:34,109 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:48:39,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:48:44,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:48:46,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:46,416 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:48:46,416 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:48:47,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:48,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:48:48,023 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:48:48,023 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:48:48,025 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:48:48,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:49,008 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:48:49,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:48:50,009 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:48:50,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:48:55,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:48:58,519 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:49:00,520 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:03,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:49:03,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:03,128 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:49:04,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:04,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:49:04,723 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:49:04,723 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:49:04,725 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:49:05,040 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:49:05,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:05,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:06,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:07,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:08,043 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:49:10,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:15,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:17,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:18,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:49:18,125 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:49:18,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:18,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:49:18,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:49:18,611 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:49:18,613 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:49:19,054 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:49:19,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:20,056 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:49:20,167 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:21,167 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:21,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:22,058 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:49:26,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:28,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:28,519 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:49:29,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:29,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:49:29,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:49:29,867 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:49:29,870 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:49:30,066 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:49:30,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:31,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:31,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:32,068 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:49:36,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:38,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:38,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:49:38,954 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:49:39,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:40,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:49:40,315 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:49:40,315 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:49:40,317 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:49:40,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:41,076 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:49:41,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:42,078 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:49:42,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:47,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:52,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:54,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:49:54,381 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:49:54,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:55,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:55,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:49:55,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:49:55,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:49:55,797 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:49:56,090 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:49:56,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:57,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:57,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:49:58,092 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:49:58,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:49:58,520 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:49:59,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:00,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:01,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:02,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:03,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:50:08,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:50:11,335 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:11,335 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:50:11,335 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:50:12,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:12,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:50:12,699 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:50:12,699 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:50:12,701 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:50:13,107 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:50:13,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:13,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:50:14,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:15,109 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:50:15,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:16,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:17,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:18,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:19,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:50:19,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:20,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:24,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:50:28,522 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:50:29,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:50:34,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:50:39,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:50:44,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:50:45,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:45,403 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:50:45,403 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:50:46,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:46,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:50:46,759 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:50:46,759 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:50:46,761 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:50:47,133 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:50:47,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:48,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:49,135 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:50:49,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:49,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:50:50,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:51,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:52,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:55,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:50:57,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:58,525 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:50:58,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:50:59,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:50:59,311 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:50:59,311 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:50:59,328 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:50:59,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:00,160 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:51:00,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:51:00,403 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:51:00,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:00,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:01,161 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:51:05,660 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:06,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:07,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:07,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:51:07,919 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:51:07,920 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:51:07,921 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:51:08,182 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:51:08,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:09,183 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:51:09,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:10,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:10,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:11,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:12,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:13,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:14,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:15,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:51:15,402 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:51:15,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:16,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:16,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:21,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:26,496 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:28,525 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:51:29,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:30,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:51:30,402 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:51:30,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:31,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:51:31,071 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:51:31,071 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:51:31,073 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:51:31,218 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:51:31,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:32,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:32,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:33,220 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:51:33,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:37,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:42,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:47,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:51,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:51:51,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:51,255 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:51:52,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:52,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:52,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:51:52,687 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:51:52,688 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:51:52,689 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:51:53,239 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:51:53,240 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:51:53,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:54,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:51:55,241 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:51:57,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:51:58,526 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:52:03,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:52:08,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:52:13,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:52:18,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:52:23,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:52:24,230 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:52:24,230 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:24,230 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:52:25,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:25,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:52:25,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:52:25,695 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:52:25,697 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:52:26,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:26,280 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:52:27,281 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:52:28,530 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:52:28,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:52:33,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:52:38,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:52:43,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:43,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:52:43,264 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:52:44,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:44,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:52:44,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:52:44,663 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:52:44,663 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:52:44,665 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:52:45,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:45,299 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:52:45,299 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:52:46,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:47,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:47,301 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:52:48,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:49,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:50,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:50,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:52:51,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:52,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:52:55,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:52:58,529 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:53:00,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:03,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:03,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:53:03,441 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:53:04,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:04,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:53:04,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:53:04,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:53:04,797 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:53:05,318 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:53:05,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:05,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:06,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:07,320 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:53:07,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:08,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:11,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:16,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:21,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:26,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:26,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:53:26,975 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:53:27,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:27,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:28,531 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:53:28,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:53:28,551 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:53:28,551 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:53:28,553 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:53:28,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:29,383 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:53:29,384 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:53:29,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:30,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:31,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:31,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:32,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:33,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:34,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:35,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:36,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:36,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:37,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:38,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:39,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:40,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:41,973 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:53:41,974 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:53:42,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:42,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:42,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:43,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:44,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:45,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:46,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:47,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:47,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:48,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:49,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:50,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:51,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:52,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:52,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:53,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:54,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:55,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:56,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:53:56,974 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:53:57,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:57,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:58,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:53:58,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:53:58,371 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:53:58,371 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:53:58,373 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:53:58,421 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:53:58,534 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:53:58,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:53:59,422 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:53:59,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:00,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:01,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:02,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:03,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:03,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:54:08,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:54:13,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:54:16,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:16,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:54:16,899 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:54:17,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:18,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:54:18,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:54:18,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:54:18,340 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:54:18,470 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:54:18,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:19,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:54:19,470 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:54:19,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:20,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:24,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:54:28,534 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:54:29,535 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:54:30,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:31,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:31,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:54:31,898 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:54:32,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:54:32,153 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:54:32,153 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:54:32,155 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:54:32,484 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:54:32,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:33,485 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:54:33,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:34,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:35,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:54:35,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:36,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:40,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:54:45,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:54:50,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:54:55,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:54:58,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:58,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:54:58,389 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:54:58,590 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:54:59,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:54:59,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:54:59,763 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:54:59,764 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:54:59,765 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:55:00,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:00,528 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:55:00,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:01,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:01,529 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:55:02,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:05,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:10,767 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:15,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:20,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:25,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:28,537 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:55:31,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:31,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:31,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:55:31,973 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:55:32,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:33,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:55:33,487 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:55:33,487 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:55:33,489 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:55:33,556 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:55:33,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:34,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:35,575 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:55:37,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:42,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:43,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:44,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:44,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:55:44,435 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:55:44,435 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:55:44,437 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:55:44,584 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:55:45,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:45,585 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:55:46,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:46,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:55:46,972 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:55:47,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:48,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:48,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:49,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:55:53,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:58,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:55:58,541 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:56:03,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:56:03,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:03,011 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:56:03,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:56:04,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:04,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:56:04,359 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:56:04,359 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:56:04,361 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:56:04,619 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:56:05,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:05,620 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:56:06,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:07,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:08,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:09,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:56:09,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:10,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:11,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:12,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:13,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:14,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:15,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:56:15,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:16,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:17,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:18,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:56:18,009 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:56:18,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:19,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:20,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:20,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:56:21,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:22,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:23,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:24,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:25,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:26,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:56:26,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:27,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:28,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:28,541 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:56:29,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:30,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:31,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:31,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:56:32,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:33,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:56:33,009 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:56:33,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:34,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:35,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:36,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:37,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:37,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:56:38,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:39,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:40,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:41,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:42,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:42,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:56:43,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:56:43,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:43,035 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:56:43,035 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:56:43,037 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:56:43,665 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:56:44,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:45,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:45,667 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:56:46,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:47,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:48,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:56:48,009 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:56:48,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:48,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:56:49,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:50,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:51,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:52,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:53,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:54,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:56:54,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:55,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:56:58,543 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:56:59,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:04,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:05,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:57:05,215 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:57:05,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:06,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:06,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:57:06,543 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:57:06,543 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:57:06,544 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:57:06,686 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:57:07,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:07,687 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:57:08,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:09,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:14,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:19,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:23,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:57:23,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:23,634 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:57:24,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:24,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:25,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:57:25,123 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:57:25,123 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:57:25,125 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:57:25,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:25,703 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:57:26,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:27,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:27,705 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:57:28,546 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:57:28,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:29,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:30,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:30,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:31,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:32,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:33,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:34,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:35,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:40,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:43,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:57:43,861 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:57:43,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:44,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:45,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:57:45,283 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:57:45,283 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:57:45,285 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:57:45,721 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:57:45,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:57:46,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:47,722 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:57:51,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:56,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:57:58,546 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:58:01,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:58:02,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:58:02,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:02,935 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:58:03,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:04,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:58:04,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:58:04,415 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:58:04,416 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:58:04,736 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:58:04,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:05,737 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:58:05,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:06,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:07,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:58:07,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:08,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:09,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:10,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:11,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:12,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:58:13,400 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:14,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:14,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:58:14,911 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:58:14,912 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:58:14,913 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:58:15,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:15,764 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:58:16,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:17,766 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:58:18,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:58:23,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:58:28,547 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:58:29,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:58:34,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:58:35,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:58:35,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:35,568 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:58:36,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:37,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:58:37,027 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:58:37,027 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:58:37,028 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:58:37,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:37,782 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:58:38,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:39,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:39,784 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:58:40,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:58:40,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:41,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:42,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:43,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:44,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:45,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:58:50,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:58:54,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:58:54,664 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:58:54,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:55,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:55,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:58:56,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:58:56,303 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:58:56,303 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:58:56,305 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:58:56,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:58:56,812 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:58:57,813 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:58:58,550 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:59:01,552 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:06,552 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:09,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:59:09,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:59:09,770 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:59:10,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:59:11,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:59:11,094 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:59:11,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:59:11,096 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:59:11,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:59:11,827 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:59:12,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:13,829 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:59:17,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:22,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:27,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:28,550 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 07:59:29,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:59:29,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 07:59:29,534 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 07:59:30,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:59:31,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 07:59:31,219 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 07:59:31,219 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 07:59:31,221 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 07:59:31,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:59:31,845 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 07:59:32,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:32,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 07:59:33,847 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 07:59:37,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:42,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:47,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:52,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:57,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 07:59:58,551 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:00:02,553 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:07,553 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:08,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:00:08,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:08,375 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:00:09,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:10,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:00:10,015 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:00:10,015 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:00:10,018 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:00:10,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:10,891 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:00:11,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:11,893 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:00:12,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:13,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:13,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:14,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:15,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:16,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:17,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:18,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:18,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:19,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:20,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:23,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:26,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:00:26,146 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:00:26,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:27,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:27,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:00:27,547 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:00:27,547 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:00:27,549 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:00:27,920 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:00:28,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:28,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:28,553 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:00:29,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:29,922 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:00:30,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:31,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:00:34,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:39,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:44,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:49,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:54,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:58,557 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:00:59,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:00:59,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:00:59,684 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:00:59,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:00,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:01,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:01:01,131 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:01:01,131 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:01:01,148 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:01:01,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:01,946 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:01:02,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:03,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:03,948 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:01:04,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:01:04,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:05,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:06,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:07,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:08,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:09,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:01:14,685 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:01:19,685 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:01:24,685 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:01:28,557 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:01:28,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:01:28,585 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:01:28,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:29,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:29,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:01:29,966 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:01:29,967 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:01:29,967 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:01:29,969 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:01:30,010 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:01:30,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:31,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:32,012 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:01:32,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:33,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:35,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:01:40,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:01:45,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:01:46,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:01:46,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:46,428 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:01:47,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:47,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:01:47,919 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:01:47,919 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:01:47,920 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:01:48,026 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:01:48,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:49,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:01:50,028 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:01:50,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:01:55,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:01:58,558 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:02:01,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:02:05,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:02:05,073 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:02:05,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:06,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:06,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:02:06,411 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:02:06,411 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:02:06,413 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:02:07,039 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:02:07,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:07,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:02:08,040 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:02:08,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:09,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:10,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:11,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:12,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:13,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:02:13,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:14,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:15,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:16,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:17,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:18,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:02:23,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:02:28,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:02:28,561 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:02:33,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:02:38,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:02:43,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:02:43,385 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:02:43,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:43,651 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:02:44,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:44,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:02:44,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:02:44,955 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:02:44,957 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:02:45,072 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:02:45,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:46,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:02:48,075 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:02:48,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:02:53,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:02:58,561 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:02:59,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:04,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:06,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:06,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:03:06,673 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:03:07,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:08,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:03:08,491 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:03:08,492 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:03:08,493 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:03:08,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:09,091 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:03:09,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:10,092 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:03:10,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:10,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:11,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:12,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:15,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:20,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:25,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:26,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:03:26,681 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:03:26,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:27,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:28,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:03:28,287 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:03:28,287 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:03:28,289 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:03:28,562 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:03:28,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:29,111 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:03:29,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:30,112 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:03:31,564 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:36,564 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:41,565 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:46,565 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:51,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:56,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:03:58,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:03:58,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:03:58,389 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:03:58,681 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:03:59,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:00,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:04:00,027 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:04:00,027 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:04:00,029 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:04:00,157 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:04:00,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:01,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:02,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:02,158 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:04:02,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:03,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:04,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:05,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:06,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:07,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:07,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:12,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:17,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:22,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:27,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:04:27,316 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:04:27,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:27,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:28,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:28,567 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:04:28,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:04:28,691 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:04:28,691 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:04:28,692 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:04:29,194 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:04:29,194 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:04:29,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:30,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:31,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:32,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:33,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:33,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:34,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:38,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:38,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:39,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:40,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:04:40,199 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:04:40,200 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:04:40,201 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:04:40,204 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:04:40,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:41,205 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:04:41,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:42,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:04:42,315 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:04:42,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:43,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:43,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:44,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:45,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:46,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:48,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:53,493 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:57,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:04:57,875 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:04:57,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:58,567 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:04:58,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:04:58,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:04:59,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:04:59,423 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:04:59,423 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:04:59,424 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:04:59,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:00,221 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:05:00,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:01,222 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:05:01,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:02,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:03,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:03,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:05:04,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:05,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:08,876 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:05:13,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:05:18,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:05:19,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:05:19,636 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:05:19,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:20,635 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:21,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:05:21,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:05:21,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:05:21,097 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:05:21,242 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:05:21,635 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:23,244 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:05:24,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:05:28,569 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:05:29,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:05:33,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:34,635 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:05:34,636 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:05:34,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:34,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:05:35,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:05:35,191 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:05:35,191 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:05:35,193 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:05:35,273 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:05:35,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:37,275 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:05:40,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:05:45,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:05:49,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:49,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:05:49,909 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:05:50,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:51,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:05:51,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:05:51,279 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:05:51,280 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:05:51,281 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:05:51,287 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:05:51,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:52,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:53,289 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:05:53,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:54,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:05:56,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:05:58,571 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:06:00,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:01,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:01,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:06:01,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:06:01,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:06:01,797 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:06:02,298 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:06:02,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:02,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:06:03,299 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:06:03,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:04,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:04,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:06:04,908 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:06:05,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:08,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:06:13,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:06:15,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:16,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:17,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:06:17,323 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:06:17,324 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:06:17,325 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:06:17,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:18,316 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:06:18,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:19,317 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:06:19,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:06:19,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:19,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:06:19,908 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:06:20,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:21,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:22,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:23,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:24,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:24,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:06:25,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:26,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:27,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:28,572 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:06:29,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:06:31,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:32,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:32,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:06:32,910 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:06:32,910 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:06:32,911 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:06:33,361 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:06:33,361 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:06:33,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:34,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:34,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:06:34,908 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:06:35,176 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:06:35,394 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:06:35,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:36,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:37,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:38,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:39,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:40,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:40,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:06:41,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:42,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:43,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:44,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:45,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:45,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:06:46,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:47,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:48,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:51,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:06:54,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:54,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:06:54,346 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:06:55,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:55,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:06:55,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:06:55,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:06:55,928 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:06:56,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:56,411 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:06:56,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:06:57,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:57,412 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:06:58,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:06:58,573 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:06:59,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:00,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:01,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:02,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:02,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:07:03,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:04,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:05,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:06,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:07,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:08,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:07:08,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:11,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:11,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:07:11,742 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:07:12,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:13,403 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:07:13,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:07:13,443 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:07:13,443 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:07:13,444 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:07:13,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:14,426 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:07:15,427 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:07:18,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:07:23,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:07:28,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:07:28,575 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:07:32,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:07:32,827 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:07:32,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:33,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:34,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:07:34,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:07:34,147 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:07:34,147 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:07:34,148 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:07:34,446 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:07:34,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:35,446 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:07:35,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:39,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:07:44,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:07:49,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:07:51,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:07:51,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:51,883 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:07:52,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:53,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:07:53,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:07:53,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:07:53,368 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:07:53,462 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:07:53,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:07:54,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:07:55,464 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:07:58,575 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:07:59,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:04,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:09,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:14,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:19,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:24,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:26,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:26,124 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:08:26,124 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:08:27,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:27,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:08:27,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:08:27,623 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:08:27,625 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:08:28,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:28,490 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:08:28,577 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:08:29,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:29,491 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:08:30,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:30,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:35,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:40,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:41,403 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:41,403 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:08:41,403 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:08:42,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:42,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:08:42,959 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:08:42,959 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:08:42,961 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:08:43,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:43,504 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:08:44,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:45,523 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:08:45,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:50,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:54,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:55,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:56,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:08:56,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:08:56,375 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:08:56,375 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:08:56,376 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:08:56,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:08:56,402 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:08:56,531 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:08:56,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:57,532 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:08:57,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:58,578 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:08:58,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:08:59,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:00,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:01,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:01,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:09:06,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:09:11,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:09:16,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:09:21,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:09:23,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:23,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:09:23,857 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:09:24,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:25,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:09:25,211 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:09:25,211 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:09:25,213 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:09:25,570 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:09:25,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:26,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:27,213 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:09:27,572 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:09:27,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:28,581 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:09:28,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:32,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:09:37,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:09:38,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:38,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:09:38,855 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:09:39,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:39,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:09:39,771 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:09:39,771 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:09:39,788 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:09:40,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:40,584 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:09:41,586 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:09:43,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:09:48,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:49,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:49,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:09:49,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:09:49,980 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:09:49,980 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:09:49,982 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:09:50,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:50,600 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:09:51,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:51,601 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:09:52,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:09:54,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:09:58,581 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:10:00,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:05,335 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:10:05,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:05,336 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:10:05,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:06,335 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:06,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:10:06,799 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:10:06,799 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:10:06,801 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:10:07,335 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:07,614 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:10:09,616 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:10:10,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:15,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:20,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:25,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:28,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:28,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:10:28,283 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:10:28,582 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:10:29,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:29,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:10:29,663 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:10:29,663 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:10:29,664 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:10:30,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:30,647 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:10:31,648 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:10:31,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:36,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:41,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:46,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:48,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:48,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:10:48,798 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:10:49,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:50,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:10:50,295 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:10:50,295 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:10:50,297 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:10:50,684 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:10:50,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:51,685 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:10:51,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:52,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:52,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:53,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:10:57,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:10:58,585 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:11:03,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:11:08,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:11:09,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:11:09,600 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:11:09,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:10,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:11,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:11:11,111 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:11:11,111 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:11:11,113 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:11:11,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:11,716 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:11:12,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:13,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:13,718 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:11:14,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:11:14,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:15,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:16,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:17,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:18,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:19,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:11:19,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:20,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:21,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:24,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:11:28,585 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:11:29,867 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:11:29,867 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:29,867 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:11:29,868 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:11:30,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:31,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:11:31,319 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:11:31,319 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:11:31,321 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:11:31,736 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:11:31,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:33,738 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:11:35,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:11:40,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:11:42,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:43,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:43,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:11:43,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:11:43,815 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:11:43,817 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:11:44,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:44,753 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:11:44,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:11:44,866 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:11:45,754 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:11:46,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:11:51,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:11:56,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:11:58,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:58,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:11:59,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:11:59,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:11:59,547 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:11:59,547 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:11:59,549 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:11:59,766 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:11:59,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:11:59,866 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:12:00,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:01,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:01,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:01,767 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:12:06,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:11,181 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:16,181 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:16,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:12:16,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:16,525 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:12:17,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:17,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:12:17,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:12:17,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:12:17,928 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:12:18,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:18,781 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:12:19,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:19,782 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:12:20,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:21,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:21,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:22,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:23,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:24,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:25,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:26,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:28,589 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:12:31,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:36,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:41,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:43,244 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:12:43,244 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:43,244 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:12:44,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:44,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:12:44,523 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:12:44,523 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:12:44,524 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:12:44,802 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:12:45,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:45,822 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:12:47,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:52,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:57,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:12:58,590 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:12:59,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:12:59,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:12:59,577 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:13:00,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:00,835 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:13:00,836 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:13:00,836 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:13:00,837 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:13:01,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:01,836 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:13:01,836 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:13:02,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:02,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:13:03,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:03,838 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:13:04,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:05,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:08,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:13:13,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:13:18,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:13:18,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:18,045 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:13:19,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:19,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:13:19,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:13:19,515 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:13:19,515 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:13:19,516 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:13:19,853 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:13:20,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:21,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:21,855 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:13:22,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:23,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:24,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:25,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:25,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:13:26,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:27,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:28,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:28,591 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:13:29,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:30,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:30,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:13:31,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:32,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:33,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:13:33,045 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:13:33,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:34,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:35,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:36,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:36,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:13:37,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:38,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:13:38,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:38,067 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:13:38,067 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:13:38,069 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:13:38,871 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:13:39,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:39,872 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:13:40,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:41,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:42,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:42,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:13:43,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:44,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:45,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:46,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:48,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:13:53,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:13:53,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:13:53,928 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:13:53,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:54,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:55,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:13:55,323 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:13:55,323 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:13:55,324 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:13:55,886 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:13:55,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:56,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:13:57,888 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:13:58,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:13:58,592 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:14:03,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:14:08,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:08,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:14:08,928 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:14:09,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:09,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:14:09,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:14:09,679 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:14:09,679 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:14:09,680 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:14:09,901 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:14:10,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:11,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:11,903 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:14:12,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:13,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:14,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:15,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:15,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:14:16,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:17,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:18,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:19,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:21,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:14:21,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:22,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:23,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:14:23,471 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:14:23,472 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:14:23,473 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:14:23,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:23,915 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:14:23,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:14:23,928 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:14:25,930 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:14:27,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:14:28,594 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:14:32,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:14:37,596 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:14:42,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:14:42,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:42,580 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:14:42,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:14:43,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:43,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:14:43,907 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:14:43,907 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:14:43,908 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:14:43,948 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:14:44,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:45,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:45,975 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:14:46,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:47,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:48,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:14:48,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:49,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:50,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:51,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:53,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:14:58,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:14:58,191 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:14:58,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:58,595 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:14:58,596 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:14:59,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:14:59,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:14:59,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:14:59,635 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:14:59,668 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:14:59,989 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:15:00,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:01,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:01,991 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:15:02,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:03,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:04,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:04,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:15:05,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:07,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:08,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:08,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:15:08,579 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:15:08,579 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:15:08,581 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:15:08,999 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:15:09,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:09,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:15:10,000 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:15:10,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:11,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:12,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:13,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:13,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:15:13,190 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:15:14,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:15,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:15:20,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:15:25,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:15:28,597 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:15:30,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:15:30,505 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:15:30,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:30,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:15:31,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:31,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:15:31,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:15:31,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:15:31,956 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:15:32,036 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:15:32,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:33,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:34,038 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:15:34,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:35,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:15:40,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:15:45,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:15:46,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:15:46,257 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:15:46,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:47,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:47,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:15:47,690 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:15:47,690 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:15:47,691 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:15:48,052 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:15:48,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:49,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:15:50,054 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:15:51,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:15:56,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:15:58,599 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:15:59,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:00,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:00,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:16:00,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:16:00,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:16:00,928 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:16:01,065 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:16:01,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:16:01,257 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:16:01,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:02,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:02,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:16:03,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:04,068 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:16:04,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:08,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:16:13,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:16:18,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:16:19,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:16:19,446 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:16:19,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:20,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:21,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:16:21,163 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:16:21,163 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:16:21,165 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:16:21,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:22,104 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:16:22,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:23,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:24,106 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:16:24,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:16:24,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:25,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:26,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:27,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:28,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:28,599 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:16:29,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:16:29,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:30,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:34,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:16:37,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:16:37,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:37,448 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:16:38,447 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:38,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:16:38,783 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:16:38,783 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:16:38,785 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:16:39,119 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:16:39,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:16:40,120 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:16:44,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:16:49,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:16:54,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:16:55,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:55,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:16:55,828 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:16:56,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:57,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:16:57,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:16:57,636 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:16:57,637 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:16:57,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:58,138 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:16:58,601 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:16:58,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:16:59,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:00,140 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:17:00,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:17:00,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:01,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:02,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:03,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:04,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:05,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:17:05,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:06,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:07,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:08,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:09,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:10,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:17:10,827 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:17:10,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:11,095 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:17:11,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:12,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:13,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:14,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:15,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:17:15,047 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:17:15,047 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:17:15,048 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:17:15,167 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:17:15,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:16,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:17:18,170 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:17:21,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:17:25,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:25,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:17:25,827 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:17:26,457 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:27,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:17:27,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:17:27,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:17:27,399 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:17:27,400 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:17:27,457 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:28,182 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:17:28,457 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:28,604 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:17:29,457 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:30,183 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:17:30,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:31,457 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:32,457 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:32,457 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:17:33,457 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:36,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:37,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:38,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:17:38,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:17:38,067 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:17:38,067 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:17:38,069 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:17:38,222 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:17:38,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:39,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:40,224 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:17:40,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:40,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:17:40,828 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:17:41,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:42,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:43,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:17:43,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:44,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:45,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:46,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:47,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:48,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:48,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:17:49,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:52,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:53,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:53,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:17:53,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:17:53,703 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:17:53,703 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:17:53,705 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:17:54,236 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:17:54,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:55,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:55,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:17:55,828 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:17:56,238 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:17:56,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:57,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:58,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:58,604 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:17:59,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:17:59,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:04,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:04,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:05,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:06,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:18:06,223 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:18:06,223 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:18:06,224 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:18:06,248 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:18:06,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:07,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:08,250 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:18:08,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:10,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:15,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:20,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:25,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:25,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:18:25,282 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:18:25,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:26,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:26,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:18:26,964 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:18:26,964 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:18:26,965 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:18:27,275 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:18:27,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:28,275 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:18:28,605 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:18:30,277 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:18:30,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:35,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:36,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:36,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:36,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:18:36,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:18:36,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:18:36,929 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:18:37,301 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:18:37,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:38,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:39,303 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:18:39,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:40,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:18:40,281 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:18:40,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:41,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:18:41,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:46,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:51,493 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:56,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:18:58,608 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:19:01,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:19:06,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:19:07,989 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:19:07,989 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:19:08,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:08,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:09,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:19:09,463 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:19:09,463 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:19:09,465 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:19:09,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:10,331 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:19:10,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:11,332 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:19:11,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:12,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:19:12,989 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:13,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:14,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:15,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:16,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:17,989 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:19:21,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:22,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:23,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:19:23,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:19:23,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:19:23,103 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:19:23,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:23,389 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:19:23,389 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:19:23,390 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:19:24,345 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:19:25,346 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:19:28,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:19:28,608 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:19:33,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:19:38,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:19:38,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:38,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:19:38,769 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:19:39,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:40,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:19:40,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:19:40,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:19:40,368 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:19:40,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:41,368 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:19:41,368 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:19:41,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:19:44,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:19:49,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:19:54,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:19:58,610 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:19:59,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:01,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:01,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:20:01,574 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:20:02,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:03,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:20:03,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:20:03,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:20:03,097 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:20:03,388 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:20:03,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:04,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:05,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:05,389 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:20:05,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:06,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:07,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:10,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:15,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:20,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:25,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:25,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:25,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:20:25,678 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:20:26,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:27,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:20:27,159 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:20:27,159 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:20:27,161 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:20:27,421 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:20:27,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:28,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:20:28,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:29,423 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:20:29,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:30,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:30,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:35,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:40,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:41,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:41,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:20:41,690 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:20:42,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:43,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:20:43,135 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:20:43,135 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:20:43,136 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:20:43,437 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:20:43,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:44,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:45,439 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:20:45,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:46,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:46,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:47,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:48,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:49,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:50,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:51,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:51,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:52,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:53,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:54,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:55,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:56,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:20:56,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:20:56,688 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:20:56,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:57,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:58,613 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:20:58,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:20:59,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:00,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:01,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:21:01,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:01,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:21:01,787 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:21:01,787 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:21:01,789 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:21:02,466 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:21:02,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:03,466 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:21:03,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:04,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:05,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:06,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:07,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:07,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:21:08,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:09,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:10,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:11,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:21:11,689 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:21:11,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:12,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:12,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:21:13,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:14,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:15,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:21:15,519 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:21:15,519 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:21:15,520 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:21:15,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:16,493 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:21:16,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:17,494 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:21:17,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:18,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:21:18,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:19,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:20,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:21,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:22,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:23,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:24,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:21:24,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:25,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:26,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:21:26,689 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:21:26,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:27,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:21:27,360 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:21:27,360 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:21:27,362 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:21:27,517 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:21:27,517 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:21:27,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:28,614 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:21:28,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:29,519 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:21:29,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:30,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:21:35,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:21:36,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:37,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:37,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:21:37,451 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:21:37,451 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:21:37,452 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:21:37,527 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:21:38,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:39,529 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:21:41,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:21:46,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:21:51,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:21:55,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:21:55,906 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:21:55,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:56,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:57,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:21:57,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:21:57,375 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:21:57,375 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:21:57,376 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:21:57,545 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:21:57,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:21:58,615 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:21:59,547 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:22:02,616 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:22:07,617 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:22:12,617 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:22:15,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:15,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:22:15,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:22:16,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:16,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:22:16,907 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:22:16,907 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:22:16,909 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:22:17,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:17,578 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:22:17,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:22:18,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:19,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:19,579 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:22:20,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:21,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:22,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:23,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:23,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:22:24,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:25,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:26,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:27,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:28,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:28,618 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:22:28,619 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:22:29,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:30,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:22:30,366 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:22:30,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:31,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:32,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:32,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:22:32,615 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:22:32,615 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:22:32,617 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:22:33,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:33,592 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:22:33,592 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:22:34,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:34,617 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:22:35,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:36,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:37,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:40,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:22:45,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:22:47,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:47,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:22:47,744 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:22:48,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:49,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:22:49,259 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:22:49,259 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:22:49,260 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:22:49,623 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:22:49,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:50,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:51,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:22:51,625 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:22:51,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:22:56,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:22:58,618 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:23:01,619 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:23:06,619 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:23:06,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:23:06,936 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:23:06,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:07,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:08,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:23:08,427 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:23:08,427 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:23:08,429 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:23:08,640 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:23:08,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:09,641 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:23:09,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:10,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:11,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:23:11,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:12,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:16,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:23:17,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:18,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:18,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:23:18,615 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:23:18,615 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:23:18,616 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:23:18,650 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:23:19,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:19,651 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:23:20,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:22,617 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:23:26,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:23:26,663 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:23:26,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:27,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:27,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:23:28,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:23:28,191 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:23:28,191 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:23:28,192 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:23:28,619 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:23:28,660 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:23:28,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:29,661 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:23:29,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:30,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:31,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:32,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:33,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:33,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:23:38,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:23:43,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:23:48,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:23:52,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:23:52,450 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:23:52,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:53,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:53,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:23:54,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:23:54,043 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:23:54,043 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:23:54,044 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:23:54,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:54,702 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:23:55,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:55,703 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:23:56,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:57,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:58,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:58,622 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:23:59,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:23:59,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:00,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:01,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:04,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:09,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:14,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:19,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:24,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:28,621 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:24:30,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:31,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:24:31,232 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:24:31,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:32,230 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:32,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:24:32,575 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:24:32,575 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:24:32,576 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:24:32,733 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:24:33,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:33,734 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:24:34,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:35,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:36,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:36,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:37,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:38,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:39,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:40,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:41,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:41,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:42,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:43,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:46,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:48,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:48,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:24:48,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:24:49,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:49,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:24:49,575 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:24:49,575 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:24:49,576 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:24:49,763 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:24:50,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:24:51,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:51,764 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:24:56,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:24:58,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:25:01,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:25:06,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:25:11,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:25:16,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:25:21,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:21,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:25:21,274 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:25:22,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:22,561 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:25:22,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:25:22,727 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:25:22,727 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:25:22,728 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:25:22,793 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:25:23,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:23,794 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:25:24,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:25,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:26,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:27,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:28,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:28,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:25:28,626 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:25:29,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:30,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:33,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:25:38,628 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:25:41,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:25:41,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:41,614 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:25:42,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:43,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:25:43,267 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:25:43,267 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:25:43,268 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:25:43,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:43,813 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:25:44,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:25:44,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:25:45,816 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:25:49,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:25:54,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:25:58,625 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:25:59,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:01,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:26:01,724 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:26:01,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:02,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:03,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:26:03,355 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:26:03,355 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:26:03,356 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:26:03,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:03,833 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:26:04,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:05,356 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:05,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:05,835 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:26:06,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:07,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:08,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:09,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:10,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:10,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:11,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:12,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:13,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:14,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:14,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:26:14,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:26:14,956 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:26:14,957 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:26:15,871 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:26:15,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:15,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:16,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:26:16,724 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:26:16,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:17,873 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:26:17,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:18,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:19,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:20,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:21,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:21,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:22,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:26,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:28,627 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:26:28,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:29,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:30,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:26:30,443 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:26:30,443 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:26:30,444 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:26:30,886 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:26:30,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:31,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:26:31,724 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:26:31,887 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:26:31,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:31,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:32,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:33,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:34,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:35,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:36,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:36,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:37,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:38,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:39,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:40,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:41,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:42,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:42,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:43,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:44,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:45,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:26:45,171 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:26:45,171 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:26:45,186 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:26:45,899 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:26:45,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:46,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:26:46,724 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:26:46,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:47,901 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:26:47,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:47,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:48,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:52,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:57,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:26:58,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:58,628 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:26:59,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:26:59,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:26:59,523 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:26:59,523 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:26:59,525 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:26:59,912 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:27:00,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:01,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:01,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:27:01,724 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:27:01,914 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:27:02,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:03,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:27:03,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:04,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:05,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:06,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:07,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:08,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:27:08,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:09,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:10,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:11,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:12,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:13,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:14,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:27:14,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:15,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:16,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:16,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:27:16,724 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:27:17,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:18,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:19,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:27:19,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:20,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:21,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:22,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:27:22,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:22,143 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:27:22,143 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:27:22,145 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:27:22,933 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:27:23,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:23,934 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:27:24,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:27:28,631 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:27:29,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:27:34,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:27:39,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:27:44,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:27:49,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:27:54,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:27:58,631 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:27:59,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:27:59,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:27:59,260 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:28:00,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:00,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:00,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:28:00,642 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:28:00,643 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:28:00,644 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:28:01,007 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:28:01,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:02,008 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:28:02,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:03,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:04,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:05,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:06,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:06,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:09,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:10,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:11,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:28:11,203 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:28:11,203 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:28:11,205 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:28:11,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:12,033 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:28:12,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:12,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:14,034 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:28:17,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:22,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:27,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:28,632 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:28:28,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:28:28,699 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:28:28,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:29,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:30,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:28:30,043 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:28:30,043 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:28:30,045 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:28:30,047 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:28:30,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:31,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:32,049 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:28:32,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:33,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:33,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:34,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:35,700 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:38,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:43,700 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:48,700 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:53,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:56,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:56,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:28:56,682 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:28:57,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:58,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:28:58,155 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:28:58,155 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:28:58,156 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:28:58,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:28:58,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:28:59,100 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:28:59,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:28:59,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:00,101 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:29:00,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:01,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:02,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:03,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:04,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:04,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:29:05,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:09,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:29:14,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:29:17,511 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:29:17,511 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:17,511 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:29:18,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:19,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:29:19,139 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:29:19,139 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:29:19,140 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:29:19,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:20,120 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:29:20,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:29:20,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:21,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:22,121 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:29:22,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:23,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:24,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:25,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:29:28,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:29:30,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:29:35,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:29:40,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:29:41,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:41,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:29:41,274 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:29:42,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:43,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:29:43,011 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:29:43,011 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:29:43,013 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:29:43,140 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:29:43,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:29:46,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:29:46,143 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:29:51,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:29:56,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:29:58,636 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:30:01,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:01,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:30:01,342 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:30:01,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:30:02,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:02,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:30:02,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:30:02,867 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:30:02,868 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:30:03,169 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:30:03,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:04,170 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:30:04,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:05,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:06,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:07,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:30:07,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:08,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:12,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:30:15,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:16,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:30:16,340 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:30:16,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:17,155 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:30:17,156 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:30:17,156 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:30:17,158 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:30:17,214 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:30:17,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:18,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:30:18,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:19,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:20,217 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:30:20,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:21,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:22,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:23,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:30:23,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:24,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:25,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:26,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:27,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:28,638 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:30:28,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:29,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:30:34,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:30:39,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:30:44,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:30:48,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:48,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:30:48,524 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:30:49,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:49,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:30:50,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:30:50,467 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:30:50,467 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:30:50,468 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:30:50,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:51,242 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:30:51,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:30:52,243 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:30:55,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:30:58,639 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:31:00,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:31:05,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:31:07,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:07,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:31:07,745 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:31:08,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:09,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:31:09,451 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:31:09,451 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:31:09,453 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:31:09,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:10,258 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:31:10,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:11,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:31:11,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:12,260 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:31:12,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:13,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:14,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:15,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:16,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:31:21,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:22,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:22,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:31:22,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:31:22,744 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:31:22,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:31:23,004 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:31:23,004 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:31:23,005 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:31:23,272 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:31:23,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:24,273 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:31:24,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:26,275 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:31:28,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:31:28,640 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:31:33,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:31:35,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:36,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:37,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:31:37,147 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:31:37,147 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:31:37,149 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:31:37,286 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:31:37,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:37,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:31:37,744 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:31:38,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:39,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:31:39,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:40,289 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:31:40,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:41,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:44,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:31:49,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:31:53,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:53,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:31:53,850 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:31:54,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:55,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:31:55,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:31:55,287 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:31:55,287 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:31:55,289 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:31:55,316 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:31:55,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:31:58,319 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:31:58,642 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:32:00,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:05,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:10,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:12,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:12,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:32:12,552 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:32:13,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:14,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:32:14,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:32:14,039 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:32:14,041 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:32:14,335 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:32:14,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:15,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:16,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:16,337 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:32:16,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:17,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:21,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:26,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:28,645 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:32:31,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:36,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:38,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:32:38,204 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:38,204 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:32:39,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:39,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:32:39,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:32:39,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:32:39,940 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:32:40,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:40,372 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:32:41,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:41,373 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:32:41,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:46,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:51,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:54,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:54,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:32:54,248 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:32:55,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:55,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:32:55,775 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:32:55,775 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:32:55,776 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:32:56,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:56,400 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:32:57,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:57,402 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:32:57,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:32:58,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:32:58,644 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:32:59,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:03,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:33:08,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:33:13,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:33:18,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:33:23,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:33:23,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:23,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:33:23,882 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:33:24,880 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:25,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:33:25,331 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:33:25,331 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:33:25,333 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:33:25,425 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:33:25,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:26,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:27,427 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:33:27,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:28,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:33:28,646 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:33:28,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:29,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:33,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:33:38,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:33:41,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:41,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:33:41,627 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:33:42,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:43,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:33:43,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:33:43,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:33:43,196 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:33:43,457 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:33:43,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:44,197 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:33:44,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:45,474 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:33:45,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:46,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:47,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:49,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:33:53,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:54,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:54,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:33:54,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:33:54,559 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:33:54,561 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:33:55,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:55,484 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:33:55,485 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:33:55,561 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:33:56,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:56,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:33:56,626 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:33:57,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:58,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:33:58,649 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:33:59,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:00,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:00,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:34:01,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:02,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:03,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:04,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:05,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:06,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:06,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:34:07,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:08,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:09,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:10,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:11,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:11,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:34:11,626 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:34:11,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:34:12,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:13,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:34:13,115 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:34:13,115 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:34:13,117 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:34:13,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:13,501 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:34:15,503 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:34:17,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:34:22,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:34:27,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:34:27,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:27,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:34:27,812 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:34:28,648 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:34:28,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:29,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:34:29,350 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:34:29,350 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:34:29,351 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:34:29,546 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:34:29,547 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:34:29,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:30,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:31,548 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:34:31,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:32,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:34:32,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:33,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:34,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:37,812 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:34:42,812 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:34:44,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:34:44,806 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:34:44,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:45,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:46,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:34:46,199 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:34:46,199 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:34:46,201 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:34:46,576 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:34:46,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:47,577 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:34:47,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:48,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:34:48,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:49,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:50,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:51,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:52,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:34:53,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:34:58,650 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:34:59,651 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:04,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:09,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:12,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:35:12,406 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:35:12,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:13,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:13,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:35:13,847 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:35:13,847 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:35:13,849 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:35:14,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:14,603 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:35:14,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:15,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:15,618 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:35:16,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:19,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:24,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:28,651 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:35:30,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:35,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:38,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:38,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:35:38,883 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:35:39,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:40,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:35:40,244 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:35:40,244 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:35:40,246 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:35:40,656 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:35:40,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:41,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:41,657 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:35:41,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:42,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:46,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:51,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:56,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:35:56,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:35:56,896 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:35:56,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:57,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:58,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:35:58,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:35:58,415 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:35:58,416 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:35:58,653 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:35:58,706 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:35:58,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:35:59,707 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:35:59,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:00,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:01,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:36:01,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:02,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:03,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:06,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:36:10,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:11,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:11,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:36:11,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:36:11,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:36:11,812 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:36:11,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:36:11,895 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:36:12,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:36:12,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:12,720 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:36:13,721 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:36:17,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:36:22,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:36:24,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:25,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:25,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:36:25,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:36:25,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:36:25,940 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:36:26,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:26,734 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:36:26,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:36:26,896 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:36:27,735 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:36:28,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:36:28,654 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:36:33,655 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:36:38,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:36:43,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:36:44,512 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:44,513 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:36:44,513 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:36:45,511 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:46,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:36:46,031 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:36:46,031 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:36:46,033 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:36:46,511 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:46,755 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:36:47,511 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:47,756 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:36:48,511 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:49,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:36:49,512 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:50,511 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:36:54,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:36:58,656 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:36:59,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:04,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:09,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:14,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:19,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:24,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:28,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:37:28,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:28,268 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:37:28,658 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:37:29,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:29,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:29,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:37:29,763 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:37:29,763 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:37:29,764 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:37:29,805 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:37:30,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:31,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:31,807 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:37:32,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:33,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:34,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:35,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:35,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:36,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:37,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:38,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:39,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:39,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:37:39,411 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:37:39,412 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:37:39,413 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:37:39,815 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:37:40,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:40,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:41,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:41,817 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:37:42,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:43,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:37:43,267 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:37:43,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:45,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:50,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:55,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:37:58,658 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:37:59,425 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:37:59,425 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:37:59,426 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:38:00,424 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:00,660 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:38:00,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:38:00,814 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:38:00,814 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:38:00,815 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:38:00,834 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:38:01,424 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:01,835 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:38:05,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:38:09,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:10,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:10,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:38:10,771 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:38:10,771 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:38:10,772 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:38:10,844 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:38:11,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:11,773 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:38:11,845 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:38:12,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:13,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:14,424 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:38:14,425 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:38:14,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:15,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:16,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:17,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:17,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:38:18,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:19,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:20,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:21,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:22,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:23,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:38:23,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:24,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:25,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:26,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:28,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:38:28,659 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:38:30,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:38:30,677 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:38:30,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:31,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:32,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:38:32,199 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:38:32,199 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:38:32,201 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:38:32,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:32,866 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:38:33,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:33,867 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:38:34,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:38:39,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:38:44,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:38:44,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:45,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:38:45,676 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:38:45,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:46,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:38:46,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:38:46,623 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:38:46,624 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:38:46,881 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:38:46,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:47,882 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:38:47,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:48,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:49,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:38:49,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:50,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:51,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:52,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:53,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:54,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:38:54,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:55,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:56,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:57,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:58,662 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:38:58,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:38:59,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:00,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:39:00,677 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:39:00,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:00,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:01,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:02,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:39:02,115 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:39:02,115 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:39:02,116 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:39:02,896 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:39:02,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:03,897 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:39:03,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:04,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:05,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:06,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:06,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:07,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:11,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:13,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:14,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:14,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:39:14,683 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:39:14,683 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:39:14,684 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:39:14,910 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:39:15,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:15,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:39:15,677 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:39:15,926 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:39:16,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:16,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:17,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:18,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:19,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:20,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:21,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:22,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:22,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:25,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:26,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:27,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:27,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:39:27,387 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:39:27,387 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:39:27,388 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:39:27,938 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:39:27,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:28,662 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:39:28,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:29,940 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:39:29,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:32,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:37,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:42,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:47,628 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:39:47,628 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:39:47,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:47,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:48,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:49,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:39:49,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:39:49,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:39:49,341 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:39:49,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:39:49,957 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:39:51,959 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:39:53,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:58,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:39:58,663 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:40:03,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:40:05,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:40:05,362 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:40:05,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:06,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:06,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:40:06,787 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:40:06,787 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:40:06,788 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:40:06,974 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:40:07,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:07,975 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:40:08,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:08,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:40:09,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:10,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:11,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:14,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:40:19,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:40:24,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:40:24,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:24,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:40:24,526 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:40:25,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:26,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:40:26,203 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:40:26,203 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:40:26,205 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:40:26,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:26,994 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:40:27,995 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:40:28,666 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:40:29,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:40:34,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:40:37,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:38,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:39,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:40:39,090 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:40:39,090 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:40:39,091 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:40:39,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:39,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:40:39,525 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:40:39,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:40:40,008 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:40:40,009 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:40:40,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:40:42,010 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:40:44,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:40:49,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:40:54,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:40:58,665 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:41:00,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:05,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:05,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:41:05,123 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:41:06,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:06,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:06,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:41:06,583 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:41:06,583 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:41:06,584 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:41:07,030 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:41:07,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:08,031 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:41:08,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:11,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:16,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:21,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:24,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:41:24,083 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:41:24,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:25,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:25,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:41:25,711 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:41:25,711 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:41:25,712 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:41:26,057 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:41:26,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:26,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:27,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:28,059 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:41:28,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:28,667 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:41:29,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:30,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:31,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:32,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:37,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:41,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:41:41,659 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:41:41,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:42,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:42,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:43,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:41:43,191 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:41:43,191 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:41:43,192 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:41:43,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:44,075 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:41:44,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:45,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:46,076 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:41:46,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:47,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:48,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:48,660 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:49,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:53,660 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:56,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:56,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:41:56,782 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:41:57,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:58,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:41:58,227 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:41:58,227 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:41:58,229 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:41:58,668 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:41:58,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:41:58,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:41:59,090 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:41:59,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:00,091 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:42:00,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:01,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:02,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:03,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:03,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:42:04,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:05,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:08,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:42:13,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:42:17,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:17,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:42:17,388 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:42:18,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:19,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:42:19,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:42:19,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:42:19,175 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:42:19,176 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:42:19,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:20,122 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:42:20,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:21,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:22,123 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:42:22,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:23,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:24,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:24,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:42:25,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:26,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:27,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:28,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:28,671 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:42:29,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:29,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:42:30,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:31,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:32,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:42:32,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:32,388 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:42:33,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:34,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:35,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:35,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:42:36,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:37,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:38,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:39,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:40,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:41,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:42:41,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:42,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:43,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:44,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:42:44,055 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:42:44,055 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:42:44,057 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:42:44,174 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:42:44,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:45,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:46,176 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:42:46,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:47,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:42:47,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:42:47,388 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:42:47,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:48,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:49,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:50,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:51,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:52,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:42:52,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:53,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:54,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:55,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:56,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:57,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:42:57,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:58,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:42:58,671 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:42:59,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:43:00,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:43:01,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:43:02,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:43:02,019 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:43:02,019 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:43:02,021 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:43:02,191 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:43:02,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:43:02,387 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:43:02,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:43:02,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:04,193 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:43:07,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:12,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:17,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:20,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:43:20,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:43:20,462 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:43:21,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:43:22,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:43:22,011 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:43:22,011 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:43:22,012 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:43:22,210 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:43:22,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:43:23,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:23,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:43:24,212 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:43:28,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:28,672 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:43:33,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:38,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:43,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:48,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:53,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:58,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:43:58,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:43:59,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:43:59,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:43:59,069 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:44:00,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:00,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:44:00,595 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:44:00,595 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:44:00,597 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:44:01,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:01,244 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:44:02,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:02,245 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:44:03,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:04,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:44:04,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:05,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:06,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:07,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:08,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:09,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:10,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:10,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:44:11,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:12,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:13,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:14,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:44:14,067 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:44:14,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:15,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:15,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:44:16,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:17,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:18,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:19,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:20,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:21,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:44:21,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:24,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:25,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:26,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:44:26,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:44:26,445 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:44:26,445 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:44:26,448 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:44:26,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:27,267 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:44:27,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:28,268 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:44:28,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:28,675 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:44:29,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:44:29,068 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:44:29,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:30,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:31,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:31,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:44:32,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:33,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:34,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:35,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:36,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:37,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:44:37,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:44:37,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:44:37,415 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:44:37,416 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:44:37,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:38,278 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:44:38,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:39,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:40,280 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:44:40,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:41,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:42,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:44:42,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:43,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:44,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:44:44,068 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:44:44,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:45,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:46,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:47,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:48,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:44:48,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:49,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:50,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:51,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:52,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:53,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:44:54,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:44:58,678 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:44:59,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:04,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:09,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:45:09,519 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:45:09,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:09,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:10,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:11,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:45:11,163 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:45:11,163 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:45:11,165 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:45:11,339 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:45:11,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:12,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:13,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:14,342 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:45:14,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:15,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:15,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:20,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:25,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:28,678 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:45:29,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:45:29,583 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:45:29,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:30,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:30,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:31,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:45:31,479 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:45:31,479 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:45:31,481 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:45:31,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:32,358 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:45:32,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:33,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:34,360 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:45:34,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:35,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:36,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:36,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:37,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:38,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:41,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:46,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:47,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:47,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:45:47,577 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:45:48,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:49,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:45:49,377 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:45:49,377 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:45:49,378 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:45:49,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:50,376 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:45:50,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:51,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:52,378 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:45:52,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:52,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:53,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:45:57,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:45:58,679 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:46:02,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:46:07,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:46:12,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:46:15,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:15,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:46:15,866 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:46:16,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:17,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:46:17,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:46:17,611 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:46:17,613 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:46:17,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:18,430 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:46:18,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:46:18,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:19,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:20,432 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:46:20,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:21,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:22,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:23,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:46:23,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:24,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:25,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:28,680 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:46:29,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:46:34,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:46:39,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:46:44,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:46:49,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:46:54,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:46:55,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:55,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:46:55,055 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:46:56,052 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:56,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:46:56,487 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:46:56,487 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:46:56,488 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:46:57,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:57,466 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:46:57,466 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:46:58,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:46:58,682 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:46:59,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:00,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:47:00,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:01,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:05,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:47:10,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:47:15,055 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:47:20,055 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:47:24,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:24,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:47:24,437 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:47:25,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:25,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:47:25,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:47:25,947 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:47:25,947 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:47:25,948 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:47:26,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:26,524 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:47:27,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:27,525 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:47:28,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:28,684 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:47:29,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:30,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:31,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:31,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:47:32,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:33,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:34,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:37,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:47:42,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:47:42,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:42,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:47:42,828 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:47:43,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:44,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:47:44,303 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:47:44,303 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:47:44,304 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:47:44,543 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:47:44,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:45,557 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:47:45,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:48,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:47:52,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:53,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:53,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:47:53,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:47:53,723 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:47:53,723 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:47:53,725 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:47:54,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:54,566 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:47:55,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:55,568 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:47:56,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:47:58,684 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:47:58,685 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:03,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:08,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:13,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:18,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:23,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:23,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:48:23,522 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:48:23,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:24,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:24,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:48:24,961 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:48:24,961 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:48:24,962 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:48:25,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:25,606 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:48:25,607 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:48:26,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:27,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:28,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:28,685 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:48:29,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:29,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:30,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:31,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:32,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:33,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:34,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:35,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:35,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:36,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:37,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:38,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:48:38,521 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:48:38,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:39,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:40,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:40,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:41,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:41,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:48:41,575 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:48:41,575 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:48:41,577 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:48:41,621 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:48:42,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:43,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:43,623 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:48:44,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:45,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:48:46,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:51,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:56,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:48:58,689 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:49:00,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:00,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:49:00,798 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:49:01,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:02,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:49:02,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:49:02,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:49:02,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:49:02,253 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:49:02,640 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:49:02,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:03,642 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:49:03,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:07,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:49:12,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:49:17,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:49:21,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:21,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:49:21,722 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:49:22,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:22,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:49:23,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:49:23,139 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:49:23,139 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:49:23,141 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:49:23,661 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:49:23,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:24,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:25,663 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:49:25,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:28,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:49:28,688 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:49:33,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:49:36,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:36,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:49:36,722 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:49:37,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:37,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:49:37,951 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:49:37,951 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:49:37,952 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:49:38,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:38,706 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:49:38,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:49:39,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:39,707 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:49:43,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:49:48,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:49:51,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:51,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:49:51,722 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:49:52,651 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:53,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:49:53,079 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:49:53,079 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:49:53,081 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:49:53,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:53,720 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:49:54,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:49:54,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:49:55,722 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:49:58,690 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:49:59,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:04,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:06,377 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:06,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:50:06,722 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:50:07,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:07,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:50:07,891 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:50:07,891 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:50:07,893 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:50:08,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:08,735 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:50:09,736 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:50:09,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:14,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:18,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:19,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:20,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:20,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:50:20,075 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:50:20,075 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:50:20,091 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:50:20,748 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:50:20,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:21,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:50:21,723 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:50:21,749 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:50:21,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:22,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:23,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:24,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:25,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:28,691 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:50:31,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:31,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:32,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:33,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:50:33,505 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:50:33,505 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:50:33,506 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:50:33,761 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:50:33,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:34,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:35,763 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:50:35,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:36,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:50:36,723 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:50:37,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:42,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:44,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 08:50:44,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:45,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:46,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:50:46,047 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:50:46,047 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:50:46,049 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:50:46,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:46,777 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:50:47,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:47,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:47,778 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:50:48,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:49,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:50,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:51,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:51,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:50:51,722 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:50:52,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:50:52,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:57,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:50:58,692 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:51:00,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:01,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:01,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:51:01,581 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:51:01,581 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:51:01,582 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:51:01,803 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:51:01,803 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:51:02,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:03,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:03,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:51:03,805 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:51:04,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:05,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:06,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:06,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:51:06,723 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:51:07,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:08,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:09,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:51:09,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:14,020 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:51:19,020 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:51:24,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:51:27,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:27,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:51:27,524 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:51:28,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:28,693 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:51:28,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:51:28,951 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:51:28,951 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:51:28,952 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:51:29,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:29,858 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:51:29,858 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:51:29,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:51:30,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:31,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:32,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:35,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:51:40,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:51:45,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:51:50,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:51:55,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:51:58,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:51:59,456 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:51:59,456 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:51:59,456 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:52:00,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:00,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:01,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:52:01,087 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:52:01,087 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:52:01,091 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:52:01,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:01,915 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:52:03,917 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:52:06,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:11,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:16,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:16,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:52:16,088 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:52:16,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:17,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:17,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:52:17,631 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:52:17,631 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:52:17,633 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:52:17,944 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:52:18,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:19,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:19,946 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:52:20,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:21,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:22,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:22,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:23,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:27,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:28,712 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:52:32,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:37,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:42,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:47,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:52,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:53,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:52:53,064 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:52:53,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:54,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:54,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:52:54,711 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:52:54,711 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:52:54,713 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:52:54,975 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:52:55,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:55,976 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:52:56,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:57,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:58,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:52:58,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:52:58,697 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:52:59,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:03,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:53:04,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:05,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:05,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:53:05,387 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:53:05,387 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:53:05,389 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:53:05,990 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:53:06,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:07,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:07,993 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:53:08,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:08,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:53:08,063 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:53:09,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:09,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:53:14,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:53:14,377 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:15,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:15,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:53:15,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:53:15,955 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:53:15,970 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:53:16,005 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:53:16,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:17,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:18,007 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:53:18,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:19,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:53:24,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:53:28,699 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:53:30,700 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:53:35,700 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:53:40,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:53:45,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:45,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:53:45,220 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:53:46,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:46,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:53:46,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:53:46,739 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:53:46,739 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:53:46,741 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:53:47,033 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:53:47,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:48,034 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:53:48,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:49,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:50,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:53:52,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:53:57,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:53:58,701 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:54:02,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:54:03,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:54:03,818 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:54:03,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:04,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:05,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:54:05,183 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:54:05,183 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:54:05,185 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:54:05,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:06,053 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:54:08,055 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:54:08,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:54:13,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:54:18,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:54:23,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:54:28,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:54:28,701 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:54:33,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:54:33,627 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:54:33,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:33,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:54:34,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:35,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:54:35,207 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:54:35,207 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:54:35,208 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:54:35,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:36,082 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:54:36,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:37,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:38,085 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:54:38,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:39,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:39,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:54:40,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:44,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:54:46,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:47,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:47,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:54:47,758 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:54:47,758 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:54:47,760 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:54:48,098 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:54:48,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:48,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:54:48,627 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:54:49,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:49,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:54:50,100 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:54:50,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:51,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:54:55,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:54:58,703 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:55:00,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:04,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:55:04,807 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:55:04,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:05,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:06,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:06,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:55:06,275 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:55:06,275 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:55:06,277 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:55:06,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:07,121 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:55:07,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:08,122 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:55:11,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:16,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:21,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:26,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:28,704 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:55:31,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:35,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:35,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:55:35,387 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:55:36,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:36,966 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:37,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:55:37,011 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:55:37,011 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:55:37,013 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:55:37,166 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:55:37,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:38,167 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:55:38,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:39,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:40,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:41,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:42,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:55:42,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:47,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:52,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:57,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:55:58,707 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:56:01,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:56:01,133 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:56:01,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:02,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:02,452 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:56:02,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:56:02,499 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:56:02,499 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:56:02,501 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:56:03,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:03,204 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:56:04,205 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:56:07,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:56:12,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:56:17,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:56:22,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:56:22,304 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:56:22,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:22,513 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:56:23,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:23,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:56:23,603 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:56:23,603 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:56:23,605 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:56:24,237 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:56:24,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:25,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:26,239 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:56:26,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:27,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:28,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:56:28,706 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:56:33,708 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:56:38,708 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:56:43,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:56:48,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:56:52,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:52,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:56:52,128 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:56:53,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:53,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:56:53,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:56:53,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:56:53,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:56:53,941 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:56:54,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:54,263 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:56:55,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:56:56,265 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:56:58,708 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:56:59,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:04,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:09,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:12,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:57:12,299 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:57:12,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:13,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:13,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:57:13,691 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:57:13,691 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:57:13,692 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:57:14,282 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:57:14,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:15,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:15,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:16,284 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:57:16,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:17,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:18,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:21,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:24,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:25,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:25,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:57:25,975 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:57:25,975 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:57:25,976 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:57:26,293 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:57:26,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:26,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:27,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:57:27,299 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:57:27,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:28,295 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:57:28,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:28,711 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:57:29,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:30,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:31,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:32,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:37,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:38,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:39,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:39,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:57:39,467 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:57:39,467 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:57:39,469 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:57:40,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:40,307 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:57:41,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:42,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:42,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:57:42,299 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:57:42,308 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:57:42,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:43,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:44,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:45,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:46,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:47,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:48,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:48,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:49,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:50,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:51,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:52,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:53,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:57,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:57,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:57:57,714 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:57:58,710 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:57:58,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:57:58,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:57:59,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:57:59,127 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:57:59,127 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:57:59,129 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:57:59,325 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:57:59,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:00,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:01,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:02,328 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:58:02,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:03,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:03,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:58:04,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:05,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:06,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:08,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:58:12,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:12,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:58:12,713 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:58:13,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:13,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:58:13,583 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:58:13,584 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:58:13,585 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:58:14,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:14,338 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:58:14,338 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:58:14,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:58:15,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:16,340 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:58:19,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:58:24,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:58:28,712 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:58:29,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:58:32,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:58:32,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:32,826 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:58:33,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:34,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:58:34,319 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:58:34,319 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:58:34,321 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:58:34,356 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:58:34,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:35,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:58:36,358 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:58:40,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:58:45,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:58:50,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:58:55,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:58:55,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:58:55,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:55,954 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:58:56,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:57,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:58:57,519 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:58:57,519 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:58:57,520 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:58:57,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:58:58,392 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:58:58,714 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:59:00,394 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:59:00,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:05,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:10,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:10,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:59:10,953 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:59:11,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:11,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:11,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:59:11,599 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:59:11,600 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:59:11,601 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:59:12,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:12,405 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:59:14,407 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:59:16,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:21,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:26,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:28,714 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:59:30,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:30,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:59:30,209 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:59:31,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:31,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:31,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:59:31,871 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:59:31,871 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:59:31,873 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:59:32,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:32,422 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:59:33,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:34,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:34,424 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:59:35,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:37,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:42,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:47,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:51,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:51,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 08:59:51,718 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 08:59:52,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:52,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:53,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 08:59:53,355 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 08:59:53,355 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 08:59:53,357 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 08:59:53,441 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 08:59:53,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:54,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:55,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:56,444 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 08:59:56,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:57,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:58,716 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 08:59:58,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 08:59:58,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 08:59:59,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:03,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:00:08,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:00:13,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:00:18,204 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:00:18,204 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:00:18,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:19,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:19,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:00:19,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:00:19,631 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:00:19,631 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:00:19,633 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:00:20,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:20,477 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:00:21,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:22,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:22,479 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:00:23,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:25,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:00:28,717 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:00:29,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:30,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:30,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:00:30,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:00:30,603 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:00:30,603 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:00:30,605 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:00:31,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:31,488 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:00:32,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:32,489 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:00:33,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:33,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:00:33,203 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:00:34,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:35,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:36,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:36,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:00:37,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:38,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:39,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:40,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:41,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:41,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:00:42,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:43,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:44,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:45,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:46,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:00:47,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:48,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:00:48,204 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:00:48,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:49,230 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:00:49,231 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:00:49,231 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:00:49,233 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:00:49,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:49,518 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:00:50,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:51,233 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:00:51,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:52,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:52,521 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:00:53,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:54,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:00:57,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:00:58,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:00:59,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:00,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:00,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:01:00,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:01:00,815 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:01:00,817 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:01:01,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:01,529 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:01:02,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:02,530 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:01:02,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:01:03,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:01:03,204 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:01:03,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:04,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:05,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:06,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:07,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:08,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:08,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:01:09,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:10,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:11,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:12,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:13,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:13,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:01:14,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:15,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:16,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:17,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:18,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:01:18,204 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:01:18,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:18,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:01:19,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:20,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:21,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:22,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:23,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:24,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:01:24,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:25,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:26,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:27,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:27,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:01:27,929 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:01:27,929 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:01:27,930 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:01:28,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:28,552 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:01:28,720 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:01:29,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:01:29,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:30,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:30,553 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:01:31,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:32,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:33,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:01:33,204 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:01:33,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:34,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:35,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:01:40,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:01:45,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:01:46,967 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:47,966 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:48,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:01:48,204 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:01:48,357 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:01:48,442 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:01:48,443 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:01:48,444 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:01:48,584 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:01:48,967 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:49,585 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:01:49,967 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:01:50,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:01:55,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:01:58,721 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:02:00,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:02:01,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:02,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:03,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:02:03,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:02:03,415 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:02:03,633 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:02:03,633 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:02:03,635 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:02:03,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:04,600 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:02:04,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:05,601 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:02:05,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:06,635 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:02:06,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:07,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:08,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:11,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:02:16,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:02:21,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:02:22,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:22,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:02:22,381 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:02:23,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:23,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:02:23,855 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:02:23,855 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:02:23,856 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:02:24,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:24,619 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:02:25,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:25,620 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:02:26,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:27,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:02:28,724 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:02:33,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:02:36,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:37,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:37,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:02:37,380 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:02:37,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:02:37,637 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:02:37,638 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:02:37,639 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:02:38,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:38,632 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:02:39,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:39,633 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:02:39,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:02:40,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:02:44,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:02:49,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:02:54,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:02:58,723 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:02:59,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:01,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:01,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:03:01,625 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:03:02,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:03,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:03:03,167 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:03:03,167 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:03:03,169 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:03:03,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:03,653 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:03:04,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:05,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:05,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:05,655 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:03:06,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:07,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:08,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:09,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:10,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:15,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:20,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:25,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:28,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:28,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:03:28,550 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:03:28,801 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:03:29,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:30,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:03:30,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:03:30,003 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:03:30,004 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:03:30,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:30,675 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:03:31,005 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:31,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:31,675 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:03:32,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:33,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:34,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:35,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:36,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:41,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:46,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:03:46,526 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:03:46,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:46,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:47,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:48,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:03:48,051 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:03:48,051 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:03:48,052 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:03:48,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:48,692 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:03:49,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:49,693 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:03:50,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:51,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:52,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:52,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:53,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:54,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:55,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:56,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:57,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:58,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:03:58,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:03:58,727 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:03:59,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:00,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:01,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:04:01,526 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:04:01,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:02,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:03,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:03,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:04:04,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:05,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:06,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:07,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:08,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:09,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:04:09,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:10,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:11,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:12,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:12,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:04:12,851 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:04:12,851 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:04:12,853 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:04:13,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:13,713 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:04:13,714 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:04:14,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:14,853 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:04:19,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:04:24,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:04:28,727 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:04:30,728 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:04:35,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:04:39,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:04:39,194 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:04:39,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:40,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:40,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:04:40,531 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:04:40,531 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:04:40,533 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:04:40,737 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:04:41,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:41,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:04:41,738 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:04:42,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:43,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:44,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:45,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:46,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:47,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:47,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:04:48,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:49,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:50,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:53,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:53,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:04:54,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:54,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:04:54,193 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:04:54,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:04:54,687 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:04:54,688 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:04:54,689 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:04:54,752 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:04:55,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:55,753 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:04:56,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:57,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:58,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:04:58,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:04:58,728 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:05:03,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:05:08,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:05:13,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:05:16,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:16,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:05:16,766 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:05:17,764 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:18,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:05:18,319 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:05:18,319 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:05:18,320 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:05:18,764 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:18,777 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:05:19,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:05:19,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:19,778 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:05:20,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:21,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:22,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:23,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:24,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:24,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:05:25,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:26,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:27,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:28,729 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:05:28,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:30,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:05:35,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:05:37,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:05:37,414 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:05:37,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:38,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:38,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:05:38,855 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:05:38,855 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:05:38,857 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:05:39,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:39,801 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:05:39,802 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:05:40,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:40,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:05:41,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:45,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:05:50,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:05:55,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:55,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:05:55,269 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:05:56,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:56,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:05:56,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:05:56,843 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:05:56,843 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:05:56,845 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:05:57,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:57,820 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:05:57,820 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:05:58,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:05:58,733 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:05:59,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:00,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:02,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:06:07,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:06:12,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:06:16,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:06:16,408 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:06:16,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:17,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:17,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:06:17,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:06:17,855 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:06:17,855 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:06:17,856 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:06:18,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:18,846 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:06:19,847 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:06:22,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:06:27,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:06:28,732 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:06:33,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:06:33,621 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:06:33,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:33,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:06:34,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:34,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:06:34,999 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:06:34,999 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:06:35,002 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:06:35,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:35,865 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:06:35,865 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:06:39,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:06:44,003 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:06:48,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:48,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:06:48,621 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:06:49,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:49,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:06:49,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:06:49,535 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:06:49,535 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:06:49,536 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:06:49,893 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:06:50,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:51,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:06:51,895 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:06:54,537 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:06:58,734 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:06:59,735 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:04,735 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:08,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:07:08,577 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:07:08,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:09,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:09,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:09,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:07:09,919 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:07:09,919 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:07:09,922 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:07:10,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:10,912 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:07:11,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:11,913 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:07:12,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:13,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:14,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:15,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:15,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:16,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:20,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:25,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:28,737 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:07:30,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:35,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:40,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:45,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:50,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:52,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:52,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:07:52,410 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:07:53,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:53,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:07:53,871 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:07:53,871 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:07:53,874 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:07:53,946 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:07:54,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:55,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:55,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:07:55,948 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:07:56,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:57,409 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:07:58,737 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:08:01,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:08:06,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:08:11,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:08:11,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:11,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:08:11,780 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:08:12,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:13,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:08:13,567 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:08:13,567 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:08:13,569 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:08:13,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:13,967 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:08:13,967 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:08:14,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:17,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:08:22,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:08:25,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:26,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:08:26,779 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:08:26,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:27,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:08:27,225 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:08:27,225 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:08:27,227 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:08:27,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:27,982 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:08:27,982 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:08:28,227 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:08:28,738 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:08:28,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:29,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:30,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:31,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:32,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:33,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:33,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:08:34,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:35,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:36,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:37,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:38,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:39,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:08:39,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:40,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:41,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:08:41,779 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:08:41,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:42,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:43,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:44,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:45,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:08:45,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:46,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:47,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:48,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:08:48,535 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:08:48,535 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:08:48,537 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:08:48,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:49,003 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:08:49,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:50,004 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:08:50,537 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:08:50,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:51,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:52,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:53,852 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:54,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:55,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:08:55,852 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:08:58,741 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:09:01,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:09:06,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:09:07,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:07,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:09:07,941 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:09:08,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:09,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:09:09,359 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:09:09,359 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:09:09,360 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:09:09,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:10,025 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:09:10,025 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:09:12,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:09:17,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:09:18,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:19,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:20,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:09:20,263 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:09:20,263 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:09:20,265 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:09:20,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:21,038 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:09:21,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:22,039 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:09:22,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:22,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:09:22,941 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:09:23,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:09:23,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:24,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:25,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:26,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:27,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:28,741 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:09:28,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:09:28,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:29,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:30,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:33,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:09:35,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:36,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:37,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:09:37,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:09:37,459 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:09:37,462 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:09:37,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:37,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 09:09:37,941 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: stop_status +2024-09-22 09:09:38,055 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:09:38,056 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:09:39,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:09:44,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:09:49,196 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:09:54,197 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:09:58,742 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:09:59,291 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:09:59,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:09:59,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:00,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:00,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:10:00,779 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:10:00,779 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:10:00,780 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:10:01,080 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:10:01,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:02,081 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:10:02,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:04,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:09,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:14,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:19,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:24,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:28,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:10:28,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:28,743 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:10:29,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:29,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:30,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:10:30,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:10:30,003 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:10:30,004 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:10:30,108 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:10:30,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:31,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:32,110 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:10:32,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:35,005 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:40,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:42,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:10:42,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:43,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:44,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:10:44,079 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:10:44,079 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:10:44,080 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:10:44,125 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:10:44,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:45,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:45,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:46,127 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:10:46,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:47,535 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:48,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:49,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:50,535 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:55,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:10:55,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:56,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:56,295 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:10:57,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:10:57,011 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:10:57,011 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:10:57,012 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:10:57,141 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:10:57,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:58,142 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:10:58,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:10:58,746 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:10:59,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:00,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:01,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:02,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:02,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:11:03,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:04,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:05,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:06,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:07,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:07,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:11:08,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:09,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:11:09,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:09,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:11:09,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:11:09,383 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:11:10,157 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:11:10,157 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:11:10,324 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:11,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:12,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:13,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:11:13,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:14,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:15,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:16,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:17,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:18,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:19,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:11:19,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:20,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:21,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:22,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:23,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:24,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:25,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:11:25,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:26,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:27,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:28,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:28,746 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:11:29,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:30,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:31,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:11:31,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:32,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:33,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:34,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:35,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:36,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:11:41,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:11:46,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:11:51,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:11:51,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:11:51,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:52,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:53,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:11:53,374 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:11:53,374 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:11:53,376 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:11:53,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:54,205 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:11:54,205 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:11:54,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:55,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:56,377 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:11:56,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:57,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:11:58,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:11:58,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:01,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:12:02,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:03,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:04,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:12:04,563 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:12:04,563 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:12:04,565 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:12:04,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:05,218 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:12:05,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:06,219 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:12:06,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:07,565 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:12:07,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:08,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:09,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:10,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:11,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:12,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:12:15,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:16,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:16,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:12:16,735 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:12:16,735 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:12:16,736 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:12:17,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:17,233 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:12:18,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:18,233 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:12:18,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:12:19,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:20,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:21,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:24,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:12:28,750 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:12:29,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:12:29,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:12:29,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:30,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:31,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:12:31,123 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:12:31,123 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:12:31,125 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:12:31,248 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:12:31,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:32,248 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:12:32,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:12:35,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:12:40,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:12:45,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:12:50,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:12:55,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:12:58,750 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:13:00,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:13:05,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:13:06,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:13:06,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:07,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:08,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:13:08,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:13:08,235 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:13:08,237 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:13:08,286 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:13:08,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:10,289 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:13:11,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:13:16,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:13:21,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:13:23,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:13:23,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:24,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:25,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:13:25,075 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:13:25,075 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:13:25,077 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:13:25,308 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:13:25,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:26,308 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:13:26,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:27,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:13:27,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:28,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:28,752 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:13:29,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:30,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:31,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:32,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:32,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:13:33,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:34,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:35,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:36,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:38,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:13:43,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:13:48,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:13:52,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:13:52,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:53,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:53,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:13:53,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:13:53,991 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:13:53,991 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:13:53,993 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:13:54,337 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:13:54,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:13:56,338 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:13:58,754 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:13:59,755 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:03,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:14:03,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:04,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:04,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:14:04,795 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:04,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:14:04,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:14:04,797 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:14:05,348 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:14:05,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:06,349 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:14:06,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:07,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:09,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:14,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:18,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:14:18,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:19,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:19,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:20,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:14:20,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:20,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:14:20,955 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:14:20,957 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:14:21,364 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:14:21,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:22,365 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:14:22,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:23,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:25,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:28,754 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:14:31,755 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:36,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:14:36,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:37,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:37,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:38,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:14:38,031 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:14:38,031 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:14:38,032 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:14:38,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:38,381 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:14:39,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:40,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:40,383 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:14:41,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:43,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:48,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:53,343 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:57,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:14:57,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:58,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:14:58,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:14:58,755 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:14:59,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:14:59,358 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:14:59,359 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:14:59,360 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:14:59,403 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:14:59,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:00,404 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:15:00,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:01,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:02,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:03,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:04,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:04,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:15:05,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:06,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:07,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:08,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:10,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:15:12,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:13,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:14,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:15:14,299 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:15:14,299 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:15:14,301 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:15:14,420 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:15:14,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:15,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:16,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:15:16,421 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:15:16,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:17,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:18,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:19,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:20,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:21,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:21,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:15:22,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:23,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:24,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:25,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:26,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:15:26,287 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:15:26,287 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:15:26,288 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:15:26,431 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:15:26,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:27,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:15:27,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:28,433 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:15:28,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:28,757 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:15:29,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:30,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:31,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:32,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:32,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:15:33,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:34,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:35,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:36,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:37,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:38,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:38,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:15:39,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:40,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:15:40,140 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:15:40,140 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:15:40,142 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:15:40,447 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:15:40,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:41,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:42,449 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:15:42,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:43,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:43,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:15:44,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:45,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:46,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:47,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:48,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:49,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:49,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:15:50,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:51,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:52,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:53,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:54,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:55,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:55,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:15:56,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:57,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:58,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:15:58,758 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:15:59,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:00,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:00,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:16:01,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:02,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:03,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:16:03,359 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:16:03,359 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:16:03,361 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:16:03,469 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:16:03,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:04,469 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:16:04,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:05,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:06,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:16:06,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:11,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:16:16,363 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:16:17,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:16:17,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:18,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:18,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:16:18,871 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:16:18,871 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:16:18,872 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:16:19,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:19,486 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:16:20,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:20,487 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:16:21,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:21,873 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:16:22,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:23,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:24,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:27,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:16:28,759 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:16:32,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:16:37,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:16:37,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:38,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:38,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:16:39,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:16:39,075 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:16:39,075 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:16:39,076 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:16:39,521 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:16:39,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:40,522 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:16:40,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:44,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:16:47,216 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:16:47,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:48,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:48,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:16:48,563 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:16:48,563 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:16:48,565 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:16:49,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:49,531 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:16:49,565 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:16:50,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:50,532 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:16:51,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:52,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:53,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:54,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:55,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:16:56,833 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:57,833 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:58,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:16:58,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:16:58,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:16:58,252 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:16:58,541 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:16:58,761 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:16:58,833 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:16:59,833 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:17:00,543 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:17:00,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:00,833 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:17:01,833 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:17:05,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:10,763 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:15,763 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:16,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:17:16,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:17:17,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:17:17,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:17:17,991 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:17:17,991 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:17:17,992 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:17:18,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:17:18,560 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:17:19,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:17:20,562 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:17:20,993 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:25,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:28,763 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:17:31,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:36,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:40,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:17:40,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:17:41,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:17:41,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:42,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:17:42,018 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:17:42,018 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:17:42,019 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:17:42,583 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:17:42,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:17:44,585 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:17:47,020 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:52,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:57,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:17:58,763 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:18:00,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:18:00,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:01,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:01,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:18:01,935 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:18:01,935 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:18:01,937 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:18:02,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:02,601 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:18:02,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:18:03,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:04,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:04,602 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:18:05,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:06,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:08,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:18:13,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:18:17,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:18:17,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:18,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:18,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:18:18,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:18:18,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:18:18,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:18:18,940 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:18:19,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:19,616 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:18:20,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:20,617 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:18:21,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:22,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:23,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:24,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:24,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:18:25,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:26,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:27,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:28,765 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:18:29,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:29,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:18:30,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:30,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:18:30,703 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:18:30,703 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:18:30,704 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:18:31,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:31,629 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:18:32,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:32,630 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:18:33,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:35,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:18:40,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:18:45,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:18:50,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:18:55,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:18:57,812 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:18:57,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:58,767 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:18:58,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:18:59,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:18:59,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:18:59,235 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:18:59,237 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:18:59,652 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:18:59,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:19:00,653 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:19:00,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:19:01,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:06,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:11,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:16,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:21,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:25,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:19:25,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:19:26,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:19:26,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:27,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:19:27,263 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:19:27,263 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:19:27,264 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:19:27,685 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:19:27,685 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:19:27,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:19:28,767 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:19:28,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:19:32,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:37,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:42,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:47,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:52,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:54,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:19:54,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:19:55,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:19:56,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:19:56,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:19:56,235 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:19:56,237 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:19:56,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:19:56,715 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:19:57,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:19:57,716 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:19:58,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:19:58,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:19:58,768 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:19:59,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:20:00,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:20:03,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:20:08,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:20:13,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:20:18,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:20:23,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:20:28,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:20:28,771 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:20:33,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:20:38,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:20:38,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:20:39,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:20:39,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:20:40,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:20:40,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:20:40,039 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:20:40,041 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:20:40,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:20:40,763 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:20:41,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:20:41,764 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:20:45,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:20:50,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:20:55,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:20:58,787 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:21:00,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:21:03,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:21:03,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:04,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:05,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:21:05,023 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:21:05,023 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:21:05,024 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:21:05,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:05,790 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:21:05,791 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:21:06,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:21:06,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:07,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:08,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:09,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:11,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:21:16,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:21:19,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:21:20,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:20,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:21,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:21:21,603 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:21:21,603 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:21:21,605 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:21:21,809 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:21:21,809 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:21:21,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:22,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:21:22,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:23,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:24,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:25,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:27,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:21:28,772 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:21:33,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:21:38,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:21:38,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:39,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:39,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:21:40,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:21:40,007 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:21:40,008 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:21:40,010 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:21:40,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:40,828 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:21:41,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:41,829 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:21:45,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:21:50,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:21:54,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:21:54,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:55,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:55,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:21:56,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:21:56,199 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:21:56,199 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:21:56,200 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:21:56,847 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:21:56,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:57,848 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:21:57,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:58,774 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:21:58,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:21:59,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:00,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:01,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:22:01,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:02,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:06,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:22:11,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:22:16,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:22:21,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:22:23,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:22:23,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:24,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:25,230 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:22:25,231 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:22:25,231 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:22:25,233 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:22:25,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:25,880 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:22:25,880 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:22:26,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:27,233 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:22:27,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:28,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:28,776 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:22:29,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:30,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:31,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:32,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:32,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:22:33,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:34,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:35,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:38,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:22:38,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:39,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:40,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:22:40,203 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:22:40,203 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:22:40,205 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:22:40,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:40,898 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:22:41,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:41,899 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:22:42,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:43,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:22:43,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:44,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:45,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:46,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:47,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:48,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:49,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:22:49,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:50,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:51,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:52,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:53,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:54,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:55,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:22:55,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:56,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:57,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:58,776 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:22:58,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:22:59,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:23:00,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:23:00,771 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:23:00,771 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:23:00,773 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:23:00,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:23:00,918 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:23:01,773 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:01,919 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:23:06,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:11,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:16,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:21,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:26,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:28,777 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:23:31,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:36,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:41,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:43,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:23:43,230 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:23:44,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:23:44,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:23:44,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:23:44,635 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:23:44,637 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:23:44,961 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:23:45,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:23:45,962 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:23:46,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:23:47,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:23:47,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:48,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:23:53,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:58,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:23:58,778 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:24:03,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:24:08,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:24:08,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:24:08,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:09,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:10,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:24:10,275 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:24:10,275 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:24:10,277 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:24:10,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:10,986 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:24:11,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:11,987 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:24:12,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:13,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:13,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:24:14,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:15,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:16,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:17,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:18,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:18,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:24:19,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:20,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:21,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:22,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:24:22,556 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:24:22,556 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:24:22,558 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:24:22,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:22,998 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:24:23,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:23,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:24:23,999 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:24:24,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:25,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:26,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:27,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:28,780 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:24:29,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:24:32,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:33,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:34,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:24:34,167 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:24:34,167 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:24:34,169 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:24:34,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:35,014 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:24:35,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:24:35,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:36,015 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:24:36,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:37,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:38,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:39,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:40,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:40,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:24:45,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:24:50,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:24:53,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:24:53,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:54,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:55,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:24:55,015 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:24:55,015 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:24:55,017 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:24:55,034 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:24:55,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:56,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:24:56,034 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:24:56,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:57,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:58,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:24:58,781 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:24:59,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:00,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:01,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:01,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:25:02,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:03,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:04,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:05,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:06,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:07,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:25:07,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:08,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:25:08,082 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:25:08,082 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:25:08,083 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:25:08,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:09,050 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:25:09,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:10,051 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:25:10,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:11,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:12,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:25:12,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:13,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:14,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:18,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:25:20,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:25:20,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:21,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:21,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:25:21,983 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:25:21,983 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:25:21,984 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:25:22,066 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:25:22,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:23,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:23,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:25:24,068 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:25:24,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:28,782 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:25:29,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:25:34,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:25:39,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:25:41,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:25:41,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:42,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:43,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:25:43,187 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:25:43,187 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:25:43,188 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:25:43,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:44,089 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:25:44,090 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:25:44,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:45,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:25:45,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:50,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:25:55,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:25:58,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:25:58,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:58,785 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:25:59,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:25:59,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:25:59,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:25:59,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:25:59,928 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:26:00,108 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:26:00,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:00,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:01,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:02,110 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:26:02,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:03,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:04,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:05,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:06,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:11,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:15,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:26:15,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:16,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:16,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:16,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:26:16,755 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:26:16,755 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:26:16,757 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:26:17,127 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:26:17,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:18,128 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:26:21,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:26,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:28,785 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:26:31,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:32,867 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:26:32,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:33,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:34,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:26:34,283 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:26:34,283 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:26:34,285 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:26:34,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:35,146 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:26:35,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:36,147 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:26:36,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:37,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:37,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:38,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:39,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:40,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:41,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:26:42,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:47,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:52,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:57,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:26:58,786 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:27:02,535 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:27:02,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:03,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:03,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:27:04,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:27:04,091 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:27:04,091 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:27:04,092 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:27:04,175 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:27:04,535 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:05,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:06,177 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:27:06,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:07,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:08,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:09,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:09,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:27:10,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:11,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:12,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:13,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:14,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:15,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:27:16,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:17,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:17,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:27:17,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:27:17,587 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:27:17,589 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:27:18,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:18,192 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:27:18,192 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:27:19,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:20,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:20,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:27:21,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:22,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:23,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:26,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:27:28,789 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:27:31,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:27:36,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:27:36,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:37,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:37,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:27:38,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:27:38,551 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:27:38,551 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:27:38,554 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:27:38,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:27:39,228 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:27:40,229 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:27:43,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:27:48,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:27:53,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:27:58,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:27:58,789 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:27:59,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:27:59,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:00,196 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:00,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:28:00,599 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:28:00,599 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:28:00,600 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:28:01,196 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:01,249 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:28:02,196 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:02,250 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:28:03,196 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:03,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:28:04,196 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:09,196 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:28:11,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:28:11,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:12,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:13,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:28:13,112 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:28:13,112 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:28:13,114 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:28:13,261 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:28:13,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:14,262 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:28:14,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:15,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:28:15,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:16,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:17,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:18,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:19,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:20,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:28:20,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:21,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:25,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:28:28,791 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:28:30,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:28:35,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:28:40,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:28:45,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:28:50,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:28:50,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:51,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:51,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:28:51,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:28:51,971 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:28:51,971 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:28:51,974 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:28:52,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:52,295 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:28:53,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:54,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:54,296 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:28:55,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:56,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:28:57,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:28:58,792 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:29:02,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:29:02,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:02,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:29:03,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:04,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:29:04,137 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:29:04,137 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:29:04,138 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:29:04,308 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:29:04,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:05,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:06,310 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:29:06,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:08,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:29:13,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:29:18,140 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:29:19,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:29:19,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:20,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:20,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:29:20,783 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:29:20,783 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:29:20,785 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:29:21,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:21,328 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:29:22,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:22,329 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:29:23,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:29:28,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:29:28,793 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:29:33,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:29:38,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:29:41,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:29:41,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:42,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:43,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:29:43,135 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:29:43,135 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:29:43,137 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:29:43,353 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:29:43,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:44,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:29:44,353 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:29:44,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:45,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:46,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:47,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:48,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:49,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:49,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:29:50,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:54,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:29:54,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:55,537 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:56,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:29:56,055 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:29:56,055 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:29:56,057 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:29:56,367 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:29:56,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:57,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:58,369 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:29:58,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:58,794 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:29:59,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:29:59,539 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:00,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:04,539 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:09,540 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:14,540 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:14,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:30:14,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:15,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:16,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:30:16,287 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:30:16,287 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:30:16,288 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:30:16,389 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:30:16,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:17,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:18,391 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:30:18,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:20,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:25,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:28,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:30:30,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:35,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:38,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:30:38,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:39,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:39,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:30:39,863 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:30:39,863 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:30:39,864 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:30:40,415 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:30:40,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:40,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:41,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:42,417 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:30:42,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:43,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:44,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:45,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:46,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:46,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:47,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:48,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:49,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:50,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:51,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:30:51,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:30:51,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:30:51,196 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:30:51,427 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:30:51,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:52,197 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:52,428 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:30:52,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:30:57,197 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:30:58,798 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:31:02,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:31:06,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:31:06,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:07,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:08,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:31:08,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:31:08,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:31:08,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:31:08,340 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:31:08,446 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:31:08,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:09,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:10,448 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:31:10,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:11,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:12,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:13,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:13,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:31:18,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:31:23,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:31:28,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:31:28,798 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:31:33,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:31:33,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:34,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:34,275 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:31:34,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:31:34,683 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:31:34,684 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:31:34,685 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:31:35,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:35,473 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:31:36,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:36,473 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:31:37,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:38,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:40,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:31:45,236 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:31:50,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:31:51,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:31:51,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:52,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:52,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:31:52,923 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:31:52,923 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:31:52,925 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:31:53,492 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:31:53,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:54,493 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:31:54,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:55,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:31:55,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:31:58,799 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:32:01,800 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:06,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:07,763 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:32:07,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:08,763 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:09,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:32:09,211 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:32:09,211 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:32:09,213 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:32:09,508 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:32:09,763 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:10,509 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:32:10,764 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:11,764 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:12,213 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:17,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:22,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:27,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:28,830 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:32:30,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:32:30,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:31,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:32,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:32:32,215 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:32:32,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:32,215 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:32:32,216 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:32:32,530 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:32:32,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:33,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:34,532 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:32:34,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:35,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:36,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:37,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:37,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:38,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:39,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:40,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:41,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:42,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:42,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:43,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:44,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:45,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:46,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:47,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:47,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:48,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:49,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:50,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:32:50,355 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:32:50,355 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:32:50,357 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:32:50,548 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:32:50,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:51,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:52,550 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:32:52,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:52,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:53,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:54,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:55,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:56,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:57,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:32:57,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:32:58,802 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:32:58,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:03,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:33:05,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:33:05,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:06,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:06,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:33:06,743 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:33:06,743 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:33:06,745 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:33:07,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:07,565 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:33:08,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:08,566 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:33:09,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:33:14,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:33:19,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:33:22,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:33:22,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:23,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:23,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:33:23,447 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:33:23,447 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:33:23,449 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:33:23,582 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:33:24,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:24,583 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:33:25,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:33:28,803 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:33:30,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:33:35,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:33:40,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:33:40,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:33:40,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:41,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:42,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:33:42,155 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:33:42,155 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:33:42,157 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:33:42,615 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:33:42,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:44,617 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:33:46,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:33:51,159 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:33:53,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:33:53,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:54,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:54,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:33:54,779 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:33:54,779 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:33:54,781 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:33:55,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:33:55,642 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:33:56,642 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:33:56,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:33:58,834 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:34:01,835 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:06,836 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:11,836 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:16,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:17,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:34:17,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:18,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:19,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:34:19,435 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:34:19,435 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:34:19,436 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:34:19,662 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:34:19,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:20,663 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:34:20,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:21,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:22,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:27,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:28,806 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:34:32,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:37,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:34:37,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:38,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:38,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:38,904 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:34:38,905 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:34:38,905 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:34:38,906 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:34:39,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:39,696 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:34:40,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:40,697 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:34:41,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:42,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:43,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:44,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:49,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:49,969 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:34:50,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:50,969 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:51,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:34:51,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:34:51,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:34:51,369 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:34:51,708 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:34:51,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:52,709 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:34:52,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:53,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:54,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:54,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:55,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:56,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:57,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:58,807 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:34:58,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:34:59,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:34:59,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:04,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:35:09,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:35:14,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:35:18,213 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:35:18,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:19,213 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:19,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:35:19,707 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:35:19,707 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:35:19,708 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:35:19,749 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:35:20,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:20,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:35:20,750 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:35:25,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:35:28,808 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:35:30,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:35:35,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:35:37,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:35:37,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:38,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:39,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:35:39,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:35:39,235 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:35:39,236 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:35:39,768 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:35:39,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:40,769 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:35:40,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:41,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:35:41,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:42,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:43,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:44,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:45,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:46,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:35:46,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:47,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:48,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:49,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:50,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:51,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:52,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:35:52,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:53,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:54,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:35:54,647 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:35:54,647 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:35:54,648 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:35:54,781 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:35:54,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:55,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:35:56,783 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:35:58,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:35:58,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:36:03,812 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:36:08,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:36:08,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:09,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:09,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:36:09,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:36:09,831 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:36:09,831 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:36:09,832 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:36:10,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:10,797 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:36:11,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:11,798 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:36:12,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:13,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:14,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:15,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:15,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:36:16,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:17,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:20,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:36:25,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:36:28,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:36:30,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:36:30,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:30,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:36:31,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:32,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:36:32,075 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:36:32,075 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:36:32,077 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:36:32,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:32,817 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:36:33,818 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:36:36,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:36:41,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:36:44,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:36:44,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:45,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:45,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:36:45,911 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:36:45,911 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:36:45,913 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:36:46,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:46,830 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:36:46,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:36:47,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:47,831 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:36:48,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:51,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:36:56,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:36:56,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:57,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:57,411 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:36:57,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:36:57,819 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:36:57,819 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:36:57,820 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:36:57,842 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:36:57,842 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:36:58,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:58,812 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:36:59,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:36:59,844 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:37:00,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:01,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:02,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:03,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:03,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:37:04,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:05,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:06,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:09,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:37:14,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:37:15,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:37:15,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:16,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:17,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:37:17,275 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:37:17,275 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:37:17,277 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:37:17,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:17,874 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:37:17,874 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:37:20,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:37:25,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:37:28,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:37:28,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:28,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:37:29,456 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:30,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:37:30,049 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:37:30,049 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:37:30,050 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:37:30,456 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:30,901 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:37:31,051 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:37:31,456 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:31,902 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:37:36,051 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:37:36,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:37:36,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:37,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:38,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:37:38,387 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:37:38,387 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:37:38,389 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:37:38,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:38,910 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:37:39,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:39,911 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:37:40,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:41,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:37:41,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:42,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:44,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:45,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:46,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:37:46,183 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:37:46,183 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:37:46,185 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:37:46,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:46,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:37:46,918 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:37:47,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:47,919 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:37:48,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:49,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:50,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:51,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:37:52,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:37:57,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:37:58,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:38:02,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:38:07,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:38:12,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:38:13,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:38:13,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:38:14,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:38:15,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:38:15,083 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:38:15,083 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:38:15,084 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:38:15,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:38:15,943 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:38:15,943 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:38:16,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:38:18,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:38:23,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:38:28,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:38:28,816 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:38:33,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:38:38,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:38:43,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:38:46,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:38:46,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:38:47,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:38:48,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:38:48,119 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:38:48,119 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:38:48,121 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:38:48,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:38:48,971 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:38:49,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:38:49,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:38:49,972 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:38:50,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:38:51,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:38:54,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:38:58,846 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:38:59,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:39:02,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:39:02,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:03,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:03,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:39:03,919 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:39:03,919 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:39:03,921 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:39:03,999 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:39:04,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:04,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:39:05,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:06,002 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:39:06,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:07,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:08,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:09,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:10,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:10,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:39:15,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:16,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:16,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:39:16,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:39:16,879 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:39:16,879 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:39:16,881 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:39:17,026 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:39:17,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:18,027 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:39:18,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:19,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:20,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:21,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:22,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:39:25,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:26,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:27,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:39:27,247 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:39:27,247 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:39:27,248 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:39:27,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:28,038 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:39:28,038 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:39:28,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:39:28,819 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:39:28,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:29,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:30,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:39:33,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:39:38,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:39:43,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:39:48,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:39:53,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:39:58,821 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:39:59,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:03,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:40:03,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:40:04,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:40:04,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:40:04,651 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:40:04,651 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:40:04,653 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:40:05,068 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:40:05,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:40:05,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:06,069 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:40:06,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:40:07,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:40:08,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:40:11,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:16,212 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:21,212 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:24,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:40:24,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:40:25,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:40:25,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:40:25,847 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:40:25,847 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:40:25,848 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:40:26,088 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:40:26,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:40:26,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:27,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:40:28,090 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:40:28,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:40:28,822 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:40:32,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:37,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:42,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:47,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:52,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:57,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:40:58,839 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:41:01,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:41:01,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:02,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:02,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:41:03,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:41:03,101 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:41:03,101 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:41:03,103 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:41:03,136 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:41:03,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:04,137 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:41:04,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:05,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:06,403 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:08,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:41:13,403 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:41:15,116 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:41:15,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:16,117 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:16,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:41:16,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:41:16,503 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:41:16,504 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:41:17,117 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:17,192 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:41:18,193 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:41:18,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:41:23,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:41:28,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:41:28,824 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:41:31,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:41:31,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:32,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:32,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:41:32,851 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:41:32,851 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:41:32,852 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:41:33,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:33,208 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:41:33,853 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:41:34,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:34,209 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:41:38,853 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:41:43,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:41:44,117 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:41:44,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:45,116 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:45,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:41:45,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:41:45,587 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:41:45,589 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:41:46,117 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:46,222 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:41:46,222 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:41:47,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:41:49,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:41:54,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:41:58,825 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:41:59,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:03,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:42:03,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:04,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:04,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:05,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:42:05,319 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:42:05,319 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:42:05,320 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:42:05,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:06,241 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:42:06,241 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:42:06,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:07,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:08,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:09,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:10,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:15,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:20,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:21,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:42:21,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:22,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:23,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:42:23,463 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:42:23,463 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:42:23,464 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:42:23,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:24,272 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:42:24,272 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:42:24,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:26,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:28,858 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:42:31,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:36,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:41,756 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:42:41,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:42,756 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:42,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:43,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:42:43,487 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:42:43,487 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:42:43,489 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:42:43,756 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:44,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:42:44,289 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:42:44,756 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:45,756 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:42:48,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:53,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:58,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:42:58,829 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:43:03,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:43:07,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:43:07,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:08,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:09,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:43:09,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:43:09,575 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:43:09,575 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:43:09,577 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:43:09,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:10,312 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:43:10,312 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:43:10,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:11,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:14,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:43:19,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:43:24,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:43:25,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:43:25,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:26,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:26,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:43:26,455 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:43:26,455 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:43:26,457 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:43:27,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:27,342 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:43:28,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:28,343 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:43:28,830 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:43:29,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:29,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:43:30,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:31,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:32,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:33,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:34,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:35,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:43:35,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:36,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:37,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:38,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:39,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:40,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:43:40,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:41,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:42,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:43,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:44,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:45,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:46,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:43:46,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:47,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:48,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:49,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:50,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:51,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:43:51,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:52,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:53,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:54,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:43:56,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:43:58,852 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:43:59,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:00,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:00,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:44:00,671 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:44:00,671 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:44:00,673 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:44:01,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:01,415 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:44:01,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:44:02,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:02,416 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:44:03,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:04,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:05,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:06,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:07,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:07,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:44:08,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:09,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:10,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:11,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:12,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:13,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:44:13,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:14,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:18,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:44:23,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:44:27,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:44:27,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:28,833 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:44:28,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:44:28,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:29,213 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:44:29,214 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:44:29,214 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:44:29,230 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:44:29,439 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:44:29,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:30,439 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:44:30,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:31,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:34,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:44:39,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:44:40,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:44:40,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:41,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:41,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:44:41,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:44:41,747 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:44:41,748 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:44:42,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:42,452 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:44:42,452 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:44:43,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:44,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:44:49,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:44:54,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:44:58,835 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:44:58,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:44:59,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:44:59,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:00,003 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:00,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:45:00,619 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:45:00,619 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:45:00,621 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:45:00,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:01,468 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:45:01,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:02,469 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:45:05,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:07,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:45:07,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:08,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:09,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:45:09,355 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:45:09,355 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:45:09,357 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:45:09,476 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:45:09,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:10,477 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:45:10,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:11,357 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:16,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:21,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:26,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:27,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:45:27,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:28,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:28,836 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:45:29,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:45:29,179 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:45:29,179 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:45:29,181 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:45:29,493 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:45:29,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:30,494 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:45:30,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:31,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:32,181 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:32,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:33,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:34,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:35,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:37,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:42,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:45:42,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:43,076 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:43,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:43,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:45:43,512 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:45:43,512 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:45:43,513 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:45:44,076 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:44,508 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:45:44,508 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:45:45,076 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:45:48,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:53,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:58,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:45:58,837 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:46:03,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:46:08,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:46:11,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:46:11,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:12,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:12,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:46:12,675 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:46:12,675 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:46:12,676 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:46:13,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:13,538 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:46:14,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:14,539 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:46:14,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:46:15,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:16,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:17,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:18,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:19,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:20,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:46:20,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:20,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:46:20,855 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:46:20,855 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:46:20,857 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:46:21,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:21,547 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:46:22,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:22,548 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:46:23,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:24,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:25,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:46:25,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:26,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:27,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:28,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:28,839 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:46:29,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:30,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:46:35,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:46:40,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:46:45,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:46:50,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:46:55,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:46:58,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:46:58,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:58,840 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:46:59,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:46:59,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:46:59,843 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:46:59,843 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:46:59,845 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:47:00,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:00,596 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:47:00,596 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:47:00,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:01,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:02,598 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:47:05,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:06,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:47:06,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:07,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:08,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:47:08,363 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:47:08,363 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:47:08,364 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:47:08,604 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:47:08,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:09,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:10,606 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:47:10,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:11,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:11,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:16,661 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:16,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:17,661 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:18,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:47:18,093 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:47:18,093 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:47:18,094 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:47:18,615 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:47:18,661 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:20,617 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:47:22,095 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:26,416 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:47:26,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:27,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:27,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:28,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:47:28,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:47:28,175 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:47:28,177 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:47:28,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:28,626 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:47:28,859 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:47:29,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:30,629 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:47:32,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:37,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:42,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:47,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:52,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:52,872 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:47:52,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:53,872 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:54,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:47:54,255 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:47:54,255 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:47:54,256 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:47:54,663 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:47:54,872 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:47:56,665 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:47:58,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:47:58,843 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:48:03,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:48:08,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:48:12,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:48:12,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:48:13,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:48:13,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:48:13,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:48:13,979 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:48:13,979 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:48:13,981 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:48:14,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:48:14,682 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:48:15,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:48:16,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:48:16,697 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:48:18,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:48:23,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:48:28,844 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:48:29,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:48:34,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:48:39,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:48:44,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:48:44,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:48:45,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:48:45,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:48:46,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:48:46,379 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:48:46,379 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:48:46,381 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:48:46,724 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:48:46,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:48:48,726 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:48:50,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:48:55,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:48:58,878 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:49:00,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:49:00,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:00,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:01,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:02,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:49:02,075 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:49:02,075 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:49:02,076 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:49:02,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:02,739 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:49:03,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:04,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:04,741 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:49:05,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:06,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:06,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:07,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:08,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:11,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:16,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:21,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:25,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:49:25,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:26,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:26,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:27,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:49:27,159 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:49:27,159 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:49:27,161 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:49:27,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:27,806 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:49:28,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:28,807 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:49:28,847 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:49:29,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:30,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:49:32,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:37,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:42,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:47,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:52,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:57,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:49:58,850 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:50:02,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:50:04,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:50:04,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:05,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:06,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:50:06,123 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:50:06,123 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:50:06,124 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:50:06,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:06,869 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:50:08,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:50:08,871 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:50:13,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:50:16,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:50:16,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:17,513 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:18,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:50:18,167 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:50:18,167 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:50:18,167 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:50:18,168 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:50:18,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:18,881 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:50:19,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:20,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:20,883 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:50:23,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:50:28,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:50:28,850 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:50:30,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:50:30,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:31,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:32,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:50:32,435 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:50:32,435 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:50:32,437 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:50:32,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:32,894 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:50:33,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:50:33,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:33,895 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:50:34,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:35,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:36,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:38,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:50:43,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:50:46,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:50:46,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:47,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:47,617 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:50:47,619 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:50:47,619 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:50:47,620 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:50:47,923 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:50:47,924 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:50:48,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:49,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:49,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:50:50,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:50:54,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:50:58,851 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:50:59,852 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:04,852 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:08,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:51:08,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:09,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:09,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:10,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:51:10,327 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:51:10,327 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:51:10,329 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:51:10,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:10,943 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:51:11,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:11,944 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:51:12,880 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:13,880 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:14,880 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:15,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:20,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:21,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:51:21,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:22,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:22,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:51:22,691 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:51:22,691 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:51:22,693 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:51:22,955 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:51:23,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:23,956 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:51:24,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:25,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:26,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:26,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:27,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:28,853 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:51:31,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:35,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:51:35,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:36,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:37,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:37,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:51:37,083 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:51:37,083 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:51:37,084 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:51:37,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:37,983 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:51:37,983 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:51:38,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:42,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:47,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:52,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:57,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:51:58,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:51:58,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:51:58,854 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:51:59,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:00,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:52:00,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:52:00,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:52:00,197 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:52:00,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:01,004 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:52:02,005 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:52:02,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:52:07,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:52:12,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:52:17,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:52:21,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:52:21,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:22,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:22,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:52:23,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:52:23,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:52:23,004 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:52:23,005 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:52:23,024 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:52:23,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:24,025 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:52:28,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:52:28,857 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:52:33,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:52:38,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:52:40,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:52:40,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:41,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:41,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:52:41,387 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:52:41,387 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:52:41,388 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:52:42,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:42,043 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:52:42,044 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:52:43,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:44,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:44,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:52:45,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:46,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:47,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:48,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:49,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:50,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:52:50,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:51,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:52,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:53,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:54,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:52:54,187 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:52:54,187 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:52:54,189 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:52:54,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:55,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:52:55,056 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:52:55,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:52:56,057 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:52:58,857 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:53:00,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:53:05,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:53:10,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:53:13,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:53:13,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:14,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:14,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:53:14,843 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:53:14,843 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:53:14,845 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:53:15,075 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:53:15,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:16,076 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:53:16,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:16,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:53:17,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:18,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:19,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:20,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:22,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:53:25,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:53:25,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:26,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:27,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:53:27,087 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:53:27,087 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:53:27,089 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:53:27,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:28,088 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:53:28,088 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:53:28,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:53:28,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:28,859 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:53:33,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:53:38,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:53:43,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:53:43,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:44,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:44,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:53:44,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:53:44,937 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:53:44,937 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:53:44,939 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:53:45,101 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:53:45,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:46,102 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:53:46,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:47,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:48,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:49,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:50,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:53:50,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:55,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:53:56,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:53:56,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:57,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:58,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:53:58,015 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:53:58,015 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:53:58,017 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:53:58,114 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:53:58,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:53:58,862 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:54:00,115 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:54:00,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:05,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:10,864 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:11,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:54:11,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:12,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:12,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:54:12,499 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:54:12,499 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:54:12,500 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:54:13,126 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:54:13,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:14,127 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:54:16,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:21,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:23,992 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:54:24,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:24,992 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:25,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:54:25,479 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:54:25,479 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:54:25,481 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:54:25,992 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:26,140 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:54:26,140 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:54:27,481 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:28,861 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:54:32,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:37,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:42,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:45,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:54:45,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:46,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:46,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:54:46,915 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:54:46,915 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:54:46,917 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:54:47,183 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:54:47,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:47,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:48,184 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:54:48,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:49,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:50,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:51,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:52,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:54:53,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:58,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:54:58,863 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:55:01,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:55:01,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:02,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:03,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:55:03,051 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:55:03,051 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:55:03,052 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:55:03,199 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:55:03,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:04,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:55:04,200 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:55:04,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:05,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:06,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:09,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:55:13,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:55:14,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:14,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:15,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:55:15,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:55:15,643 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:55:15,643 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:55:15,644 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:55:15,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:16,213 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:55:16,213 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:55:16,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:17,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:18,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:19,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:20,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:55:20,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:21,992 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:22,992 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:25,992 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:55:28,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:55:28,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:28,866 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:55:29,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:30,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:55:30,439 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:55:30,439 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:55:30,441 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:55:30,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:31,229 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:55:31,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:55:32,230 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:55:36,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:55:41,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:55:45,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:55:45,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:46,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:46,795 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:55:47,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:55:47,475 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:55:47,475 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:55:47,477 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:55:47,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:48,246 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:55:48,246 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:55:48,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:49,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:50,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:51,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:52,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:55:52,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:53,755 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:56,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:57,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:58,140 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:55:58,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:55:58,187 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:55:58,187 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:55:58,189 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:55:58,257 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:55:58,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:55:58,866 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:55:59,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:00,258 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:56:00,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:01,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:02,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:03,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:03,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:56:04,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:05,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:06,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:07,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:08,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:08,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:56:09,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:10,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:11,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:12,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:13,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:14,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:56:14,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:15,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:16,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:16,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:56:16,965 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:56:16,965 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:56:16,966 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:56:17,275 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:56:17,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:18,276 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:56:18,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:19,967 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:56:24,967 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:56:28,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:56:28,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:28,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:56:29,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:29,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:56:29,895 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:56:29,895 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:56:29,897 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:56:30,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:56:30,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:56:30,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:30,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:56:31,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:35,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:56:40,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:56:41,095 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:56:41,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:42,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:42,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:56:42,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:56:42,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:56:42,929 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:56:43,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:43,301 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:56:44,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:44,302 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:56:45,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:45,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:56:46,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:47,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:48,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:49,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:50,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:51,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:56:51,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:52,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:53,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:54,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:56:56,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:56:58,869 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:56:59,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:56:59,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:00,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:01,291 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:01,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:57:01,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:57:01,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:57:01,341 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:57:01,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:02,330 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:57:02,330 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:57:02,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:03,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:04,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:05,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:06,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:06,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:09,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:10,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:11,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:57:11,016 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:57:11,017 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:57:11,018 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:57:11,339 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:57:11,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:12,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:12,340 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:57:17,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:22,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:22,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:57:22,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:23,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:24,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:57:24,579 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:57:24,579 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:57:24,581 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:57:24,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:25,353 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:57:25,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:26,354 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:57:26,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:27,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:28,870 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:57:32,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:37,872 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:42,872 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:47,873 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:52,873 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:54,377 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:57:54,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:55,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:55,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:57:55,839 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:57:55,839 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:57:55,841 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:57:56,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:56,393 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:57:56,393 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:57:57,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:58,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:57:58,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:57:58,871 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:57:59,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:00,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:04,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:58:09,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:58:14,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:58:19,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:58:23,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:58:23,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:24,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:24,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:58:25,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:58:25,323 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:58:25,323 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:58:25,325 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:58:25,450 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:58:25,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:26,451 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:58:26,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:27,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:28,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:28,873 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:58:29,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:30,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:30,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:58:31,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:32,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:33,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:34,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:35,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:36,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:58:37,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:38,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:39,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:58:39,191 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:58:39,191 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:58:39,192 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:58:39,479 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:58:39,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:40,480 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:58:40,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:41,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:42,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:58:42,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:43,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:44,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:45,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:46,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:47,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:47,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:58:48,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:49,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:50,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:51,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:52,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:53,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:58:53,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:58:58,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:58:58,876 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:59:00,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:59:00,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:01,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:01,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:59:01,527 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:59:01,527 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:59:01,529 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:59:02,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:02,501 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:59:02,501 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:59:03,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:04,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:04,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:59:05,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:06,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:07,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:08,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:10,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:59:15,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:59:20,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:59:25,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:59:28,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 09:59:28,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:28,876 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 09:59:29,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:29,512 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 09:59:29,513 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 09:59:29,513 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 09:59:29,515 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 09:59:29,525 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 09:59:30,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 09:59:30,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:59:30,526 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 09:59:35,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:59:40,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:59:45,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:59:50,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:59:55,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 09:59:58,877 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:00:00,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:02,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:00:02,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:03,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:03,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:00:03,899 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:00:03,899 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:00:03,901 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:00:04,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:04,590 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:00:04,590 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:00:05,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:10,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:15,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:20,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:25,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:28,880 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:00:31,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:32,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:00:32,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:33,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:33,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:00:33,823 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:00:33,823 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:00:33,824 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:00:34,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:34,615 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:00:34,616 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:00:35,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:36,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:37,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:37,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:38,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:39,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:40,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:41,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:42,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:43,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:43,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:48,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:53,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:57,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:00:57,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:58,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:00:58,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:00:58,467 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:00:58,467 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:00:58,467 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:00:58,468 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:00:58,637 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:00:58,880 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:00:59,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:00,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:00,639 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:01:01,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:02,161 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:03,161 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:04,161 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:01:09,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:01:14,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:01:15,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:01:15,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:16,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:16,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:01:16,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:01:16,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:01:16,929 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:01:17,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:17,656 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:01:18,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:18,657 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:01:19,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:19,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:01:24,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:01:28,881 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:01:30,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:01:34,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:01:34,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:35,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:36,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:01:36,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:01:36,171 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:01:36,171 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:01:36,173 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:01:36,690 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:01:36,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:37,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:38,692 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:01:38,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:39,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:40,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:41,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:41,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:01:42,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:43,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:44,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:45,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:46,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:01:47,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:01:52,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:01:57,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:01:58,883 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:02:02,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:02:02,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:02:02,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:03,947 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:04,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:02:04,307 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:02:04,307 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:02:04,309 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:02:04,728 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:02:04,947 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:05,947 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:06,730 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:02:06,947 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:08,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:02:13,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:02:18,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:02:22,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:02:22,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:23,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:23,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:02:23,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:02:23,855 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:02:23,855 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:02:23,857 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:02:24,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:24,744 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:02:24,745 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:02:25,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:26,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:28,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:02:28,886 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:02:33,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:02:38,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:02:38,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:39,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:39,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:02:39,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:02:39,771 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:02:39,771 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:02:39,773 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:02:40,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:40,773 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:02:40,774 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:02:41,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:42,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:43,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:44,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:45,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:02:45,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:50,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:02:55,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:02:58,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:02:58,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:58,886 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:02:59,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:02:59,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:02:59,779 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:02:59,779 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:02:59,781 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:02:59,788 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:03:00,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:00,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:03:00,789 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:03:01,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:02,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:03,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:04,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:05,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:06,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:03:06,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:07,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:08,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:12,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:03:17,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:03:21,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:03:21,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:22,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:22,604 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:03:22,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:03:22,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:03:22,748 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:03:22,749 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:03:22,821 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:03:23,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:24,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:24,823 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:03:25,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:26,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:27,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:28,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:03:28,887 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:03:33,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:03:38,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:03:40,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:03:40,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:41,456 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:42,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:03:42,155 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:03:42,155 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:03:42,157 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:03:42,456 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:42,840 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:03:44,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:03:44,842 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:03:49,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:03:52,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:03:52,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:53,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:54,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:03:54,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:03:54,003 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:03:54,005 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:03:54,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:54,882 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:03:55,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:03:55,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:56,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:56,896 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:03:57,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:58,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:03:58,890 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:03:59,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:00,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:00,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:04:01,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:02,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:03,596 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:04,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:05,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:06,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:06,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:04:07,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:09,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:10,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:11,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:04:11,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:04:11,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:04:11,196 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:04:11,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:11,910 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:04:12,197 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:04:12,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:12,911 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:04:13,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:14,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:15,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:17,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:04:22,596 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:04:27,596 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:04:28,890 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:04:32,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:04:32,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:04:32,966 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:33,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:34,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:04:34,351 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:04:34,352 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:04:34,353 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:04:34,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:34,932 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:04:36,933 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:04:38,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:04:43,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:04:45,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:04:45,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:46,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:46,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:04:46,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:04:46,867 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:04:46,869 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:04:46,943 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:04:47,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:48,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:48,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:04:48,945 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:04:49,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:50,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:51,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:52,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:53,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:54,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:54,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:04:55,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:56,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:57,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:58,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:58,892 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:04:59,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:04:59,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:00,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:01,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:02,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:03,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:03,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:05:03,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:05:03,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:05:03,252 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:05:03,960 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:05:04,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:04,960 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:05:05,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:05,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:06,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:07,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:10,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:14,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:05:14,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:15,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:15,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:16,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:05:16,275 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:05:16,275 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:05:16,276 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:05:16,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:16,994 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:05:17,995 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:05:21,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:26,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:28,894 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:05:31,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:36,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:39,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:05:39,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:40,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:41,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:05:41,027 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:05:41,027 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:05:41,028 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:05:41,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:42,015 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:05:42,016 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:05:42,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:42,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:43,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:44,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:45,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:05:47,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:52,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:57,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:05:58,894 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:06:02,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:06:05,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:06:05,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:06,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:06,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:06:06,763 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:06:06,763 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:06:06,764 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:06:07,053 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:06:07,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:08,054 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:06:08,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:06:13,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:06:18,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:06:23,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:06:23,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:24,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:24,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:06:25,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:06:25,098 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:06:25,098 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:06:25,099 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:06:25,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:26,068 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:06:26,068 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:06:26,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:28,896 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:06:29,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:06:34,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:06:39,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:06:41,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:06:41,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:42,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:42,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:06:42,627 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:06:42,627 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:06:42,629 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:06:43,082 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:06:43,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:44,083 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:06:45,629 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:06:50,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:06:54,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:06:54,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:55,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:55,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:06:56,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:06:56,383 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:06:56,383 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:06:56,384 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:06:56,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:57,094 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:06:57,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:58,095 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:06:58,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:06:58,897 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:06:59,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:01,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:07:06,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:07:11,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:07:16,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:07:17,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:07:17,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:18,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:19,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:07:19,243 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:07:19,243 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:07:19,244 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:07:19,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:20,115 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:07:20,115 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:07:20,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:22,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:07:27,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:07:28,900 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:07:32,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:07:35,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:07:35,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:36,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:36,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:07:36,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:07:36,483 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:07:36,485 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:07:37,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:37,131 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:07:38,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:38,132 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:07:38,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:07:43,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:07:47,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:07:48,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:48,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:49,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:07:49,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:07:49,499 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:07:49,499 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:07:49,500 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:07:49,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:50,144 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:07:50,144 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:07:50,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:51,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:52,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:53,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:54,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:07:54,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:07:58,900 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:08:00,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:08:05,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:08:10,728 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:08:10,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:11,728 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:11,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:08:12,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:08:12,327 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:08:12,327 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:08:12,329 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:08:12,728 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:13,170 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:08:13,728 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:14,171 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:08:14,728 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:17,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:08:22,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:08:27,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:08:28,901 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:08:32,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:08:33,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:08:33,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:34,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:35,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:08:35,599 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:08:35,599 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:08:35,600 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:08:35,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:36,206 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:08:36,207 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:08:36,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:37,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:38,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:08:38,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:39,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:40,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:41,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:42,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:43,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:08:43,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:44,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:48,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:08:53,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:08:53,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:54,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:54,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:08:54,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:08:54,527 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:08:54,527 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:08:54,529 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:08:55,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:55,226 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:08:56,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:08:56,227 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:08:58,904 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:08:59,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:04,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:09,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:11,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:09:11,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:12,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:13,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:09:13,315 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:09:13,315 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:09:13,316 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:09:13,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:14,261 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:09:14,262 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:09:14,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:15,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:15,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:20,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:25,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:28,904 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:09:30,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:31,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:09:31,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:32,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:33,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:09:33,035 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:09:33,035 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:09:33,036 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:09:33,291 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:09:33,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:34,292 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:09:34,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:35,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:36,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:41,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:46,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:49,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:09:49,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:50,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:51,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:51,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:09:51,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:09:51,459 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:09:51,461 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:09:51,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:09:52,310 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:09:52,310 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:09:56,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:09:58,905 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:10:01,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:06,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:11,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:16,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:18,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:10:18,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:19,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:19,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:10:19,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:10:19,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:10:19,941 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:10:20,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:20,336 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:10:20,336 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:10:21,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:21,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:22,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:23,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:24,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:25,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:26,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:27,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:28,908 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:10:32,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:36,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:10:36,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:37,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:37,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:10:37,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:37,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:10:37,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:10:37,928 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:10:38,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:38,382 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:10:38,383 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:10:39,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:40,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:41,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:42,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:43,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:43,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:44,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:45,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:46,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:48,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:53,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:55,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:10:55,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:56,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:57,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:10:57,311 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:10:57,311 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:10:57,312 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:10:57,399 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:10:57,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:58,400 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:10:58,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:10:58,908 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:10:58,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:10:59,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:00,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:01,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:02,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:03,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:04,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:04,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:11:05,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:06,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:07,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:08,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:09,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:10,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:11:10,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:11,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:12,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:13,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:14,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:15,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:11:15,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:16,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:17,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:18,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:19,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:20,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:11:20,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:21,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:22,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:23,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:24,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:25,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:26,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:11:26,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:28,910 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:11:31,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:11:34,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:11:34,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:35,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:36,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:11:36,183 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:11:36,183 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:11:36,185 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:11:36,457 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:11:36,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:37,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:11:37,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:38,459 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:11:42,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:11:47,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:11:52,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:11:55,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:11:55,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:56,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:56,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:11:56,451 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:11:56,451 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:11:56,454 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:11:56,474 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:11:57,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:57,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:11:58,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:11:58,476 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:11:58,911 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:11:59,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:00,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:01,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:02,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:03,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:03,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:12:04,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:05,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:06,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:07,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:08,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:09,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:09,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:12:10,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:11,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:12,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:15,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:12:20,120 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:12:25,120 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:12:28,929 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:12:30,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:12:33,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:12:33,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:34,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:35,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:12:35,323 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:12:35,323 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:12:35,325 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:12:35,536 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:12:35,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:36,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:12:36,537 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:12:36,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:37,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:38,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:39,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:40,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:41,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:12:46,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:12:51,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:12:53,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:12:53,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:54,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:54,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:12:54,703 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:12:54,703 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:12:54,705 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:12:55,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:55,553 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:12:56,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:56,554 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:12:57,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:57,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:12:58,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:12:58,914 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:12:59,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:00,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:01,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:02,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:02,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:13:07,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:13:12,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:13:12,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:13,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:13,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:13:13,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:13:13,863 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:13:13,863 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:13:13,865 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:13:14,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:14,569 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:13:14,569 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:13:15,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:18,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:13:23,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:13:28,867 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:13:28,915 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:13:28,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:13:28,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:29,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:30,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:13:30,307 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:13:30,308 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:13:30,309 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:13:30,585 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:13:30,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:31,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:32,587 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:13:32,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:33,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:13:33,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:34,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:35,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:36,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:37,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:38,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:13:43,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:13:48,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:13:52,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:13:52,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:53,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:54,168 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:13:54,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:13:54,211 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:13:54,211 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:13:54,214 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:13:54,605 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:13:54,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:55,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:56,607 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:13:56,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:57,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:58,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:58,918 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:13:59,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:13:59,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:00,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:01,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:02,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:03,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:04,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:05,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:05,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:10,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:15,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:20,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:25,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:26,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:14:26,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:27,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:28,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:14:28,063 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:14:28,063 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:14:28,066 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:14:28,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:28,634 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:14:28,918 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:14:29,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:30,636 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:14:30,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:35,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:40,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:45,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:14:45,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:46,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:46,103 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:46,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:14:46,507 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:14:46,508 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:14:46,509 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:14:46,652 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:14:47,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:48,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:48,654 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:14:49,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:50,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:14:52,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:57,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:14:58,920 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:15:02,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:15:05,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:15:05,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:06,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:06,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:15:06,583 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:15:06,583 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:15:06,584 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:15:06,670 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:15:07,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:08,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:08,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:15:08,672 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:15:09,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:10,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:11,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:12,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:14,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:15:19,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:15:24,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:15:24,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:15:25,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:25,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:26,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:15:26,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:15:26,415 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:15:26,417 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:15:26,690 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:15:26,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:28,692 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:15:28,921 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:15:29,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:15:34,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:15:37,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:15:37,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:38,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:39,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:15:39,260 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:15:39,260 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:15:39,261 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:15:39,702 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:15:39,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:40,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:15:40,703 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:15:40,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:41,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:42,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:43,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:15:45,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:15:50,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:15:55,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:15:58,922 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:16:00,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:16:04,095 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:16:04,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:05,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:05,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:16:05,759 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:16:05,759 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:16:05,760 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:16:06,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:06,722 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:16:06,723 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:16:06,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:16:07,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:08,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:09,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:12,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:16:17,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:16:19,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:16:19,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:20,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:20,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:16:20,891 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:16:20,891 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:16:20,893 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:16:21,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:21,737 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:16:22,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:22,738 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:16:22,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:16:23,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:24,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:25,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:28,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:16:28,924 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:16:33,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:16:38,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:16:40,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:16:40,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:41,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:41,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:16:41,531 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:16:41,531 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:16:41,532 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:16:41,755 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:16:42,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:42,756 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:16:43,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:44,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:44,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:16:45,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:46,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:47,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:48,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:49,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:50,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:16:50,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:51,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:52,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:53,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:54,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:55,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:56,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:16:56,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:57,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:58,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:16:58,926 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:16:59,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:00,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:01,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:01,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:17:01,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:17:01,531 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:17:01,531 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:17:01,532 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:17:01,781 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:17:02,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:02,782 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:17:03,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:04,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:05,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:06,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:07,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:17:07,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:08,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:09,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:10,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:11,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:12,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:13,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:17:13,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:14,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:15,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:16,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:17,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:18,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:19,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:17:19,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:20,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:21,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:22,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:23,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:24,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:17:24,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:25,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:26,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:27,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:28,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:28,928 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:17:29,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:29,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:17:30,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:31,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:32,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:33,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:34,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:35,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:17:35,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:36,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:37,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:38,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:38,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:17:38,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:17:38,483 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:17:38,485 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:17:38,878 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:17:39,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:40,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:17:40,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:40,880 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:17:41,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:45,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:17:46,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:17:46,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:47,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:48,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:17:48,127 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:17:48,127 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:17:48,128 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:17:48,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:48,902 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:17:49,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:50,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:17:50,904 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:17:51,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:17:56,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:17:58,929 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:18:01,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:18:06,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:18:06,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:18:07,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:18:07,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:18:08,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:18:08,031 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:18:08,031 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:18:08,033 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:18:08,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:18:08,920 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:18:08,920 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:18:10,922 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:18:13,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:18:18,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:18:23,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:18:26,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:18:26,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:18:27,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:18:27,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:18:27,791 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:18:27,791 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:18:27,793 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:18:27,937 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:18:28,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:18:28,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:18:28,930 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:18:28,938 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:18:29,356 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:18:33,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:18:38,932 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:18:43,933 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:18:48,933 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:18:53,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:18:53,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:18:54,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:18:54,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:18:55,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:18:55,207 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:18:55,207 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:18:55,255 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:18:55,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:18:55,963 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:18:56,963 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:18:58,950 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:18:59,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:04,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:19:04,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:19:05,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:19:05,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:06,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:19:06,031 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:19:06,031 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:19:06,033 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:19:06,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:19:06,989 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:19:06,990 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:19:07,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:19:11,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:16,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:21,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:26,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:26,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:19:26,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:19:27,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:19:28,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:19:28,351 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:19:28,351 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:19:28,353 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:19:28,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:19:28,933 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:19:29,021 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:19:29,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:19:30,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:19:31,023 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:19:31,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:31,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:19:36,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:41,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:46,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:51,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:56,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:19:58,934 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:19:59,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:19:59,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:00,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:00,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:20:00,951 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:20:00,951 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:20:00,953 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:20:01,051 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:20:01,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:01,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:20:02,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:03,053 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:20:03,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:06,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:20:11,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:20:16,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:20:17,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:20:17,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:18,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:19,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:20:19,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:20:19,483 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:20:19,485 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:20:19,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:20,071 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:20:20,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:21,072 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:20:21,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:22,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:20:22,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:23,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:24,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:25,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:26,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:27,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:20:28,935 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:20:32,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:20:37,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:20:42,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:20:45,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:20:46,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:46,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:47,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:20:47,526 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:20:47,526 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:20:47,528 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:20:47,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:48,111 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:20:48,111 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:20:48,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:20:48,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:49,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:50,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:51,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:52,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:20:53,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:20:58,938 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:20:59,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:04,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:09,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:14,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:21:14,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:15,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:15,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:15,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:21:15,375 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:21:15,375 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:21:15,376 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:21:16,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:16,135 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:21:16,135 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:21:17,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:18,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:20,377 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:25,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:26,120 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:21:26,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:27,120 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:27,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:21:27,887 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:21:27,887 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:21:27,889 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:21:28,120 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:28,146 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:21:28,146 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:21:28,938 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:21:30,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:35,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:40,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:45,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:48,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:21:48,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:49,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:49,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:21:49,943 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:21:49,943 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:21:49,945 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:21:50,179 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:21:50,180 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:21:50,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:50,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:51,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:52,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:21:55,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:21:58,940 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:22:01,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:22:03,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:22:03,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:04,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:04,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:22:04,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:22:04,483 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:22:04,484 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:22:05,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:05,240 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:22:06,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:06,241 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:22:07,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:07,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:22:08,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:09,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:10,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:11,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:13,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:22:18,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:22:23,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:22:24,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:22:24,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:25,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:26,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:22:26,403 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:22:26,403 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:22:26,405 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:22:26,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:27,259 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:22:27,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:28,260 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:22:28,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:22:28,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:28,942 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:22:29,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:30,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:31,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:32,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:33,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:22:33,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:34,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:35,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:36,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:39,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:22:44,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:22:49,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:22:54,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:22:57,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:22:57,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:58,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:58,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:22:58,823 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:22:58,823 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:22:58,825 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:22:58,942 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:22:59,318 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:22:59,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:22:59,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:23:00,319 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:23:00,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:04,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:23:09,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:23:13,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:23:13,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:14,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:15,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:23:15,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:23:15,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:23:15,235 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:23:15,236 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:23:15,332 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:23:15,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:16,333 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:23:16,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:17,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:20,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:23:25,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:23:28,944 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:23:30,947 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:23:33,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:23:33,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:34,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:34,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:23:34,915 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:23:34,915 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:23:34,917 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:23:35,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:35,361 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:23:36,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:36,361 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:23:36,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:23:37,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:38,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:39,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:40,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:41,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:42,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:42,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:23:43,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:44,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:45,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:46,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:47,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:48,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:23:48,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:49,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:50,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:51,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:52,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:53,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:54,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:23:54,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:55,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:56,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:57,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:58,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:58,947 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:23:59,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:23:59,948 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:00,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:01,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:01,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:24:01,363 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:24:01,364 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:24:01,365 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:24:01,381 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:24:02,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:02,382 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:24:03,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:05,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:10,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:15,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:20,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:25,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:28,946 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:24:29,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:24:30,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:30,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:31,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:31,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:24:31,608 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:24:31,608 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:24:31,610 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:24:31,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:32,419 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:24:32,420 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:24:32,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:34,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:35,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:36,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:37,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:37,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:38,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:39,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:40,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:41,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:42,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:45,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:46,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:46,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:24:46,723 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:24:46,723 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:24:46,725 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:24:47,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:47,433 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:24:47,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:48,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:48,433 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:24:49,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:50,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:51,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:24:53,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:58,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:24:58,948 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:25:02,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:25:02,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:03,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:03,655 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:25:04,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:25:04,015 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:25:04,015 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:25:04,017 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:25:04,448 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:25:04,448 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:25:04,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:09,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:25:14,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:25:19,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:25:24,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:25:28,949 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:25:29,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:25:32,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:25:32,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:33,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:33,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:25:33,531 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:25:33,531 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:25:33,533 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:25:34,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:34,498 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:25:34,498 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:25:35,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:35,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:25:36,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:37,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:41,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:25:46,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:25:48,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:25:48,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:49,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:49,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:25:49,575 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:25:49,575 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:25:49,576 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:25:50,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:50,512 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:25:50,512 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:25:51,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:51,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:25:52,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:53,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:54,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:25:57,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:25:58,953 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:26:02,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:26:07,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:26:09,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:26:09,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:10,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:10,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:26:10,931 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:26:10,931 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:26:10,932 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:26:11,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:11,530 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:26:12,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:12,531 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:26:13,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:13,933 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:26:14,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:15,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:16,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:17,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:18,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:19,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:26:19,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:20,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:21,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:22,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:23,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:24,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:26:24,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:25,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:26,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:27,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:28,951 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:26:29,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:26:30,291 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:31,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:31,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:26:31,923 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:26:31,923 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:26:31,925 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:26:32,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:32,549 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:26:32,550 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:26:33,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:34,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:35,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:26:40,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:26:45,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:26:50,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:26:55,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:26:56,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:26:56,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:57,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:57,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:26:57,563 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:26:57,563 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:26:57,565 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:26:57,581 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:26:58,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:26:58,582 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:26:58,953 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:27:00,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:27:05,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:27:10,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:27:12,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:27:12,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:13,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:14,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:27:14,135 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:27:14,135 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:27:14,136 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:27:14,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:14,595 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:27:15,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:16,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:27:16,597 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:27:21,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:27:22,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:27:22,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:23,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:23,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:27:23,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:27:23,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:27:23,797 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:27:24,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:24,605 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:27:24,605 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:27:25,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:26,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:27:28,973 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:27:31,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:27:35,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:27:35,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:36,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:37,179 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:27:37,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:27:37,331 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:27:37,332 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:27:37,333 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:27:37,622 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:27:37,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:38,623 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:27:38,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:39,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:40,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:41,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:42,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:27:42,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:43,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:44,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:45,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:46,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:47,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:48,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:27:48,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:49,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:50,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:50,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:27:50,995 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:27:50,995 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:27:50,997 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:27:51,649 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:27:51,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:52,649 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:27:52,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:53,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:53,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:27:54,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:55,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:56,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:27:58,956 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:27:59,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:02,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:28:02,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:03,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:03,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:28:03,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:28:03,624 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:28:03,625 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:28:03,660 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:28:04,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:04,661 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:28:05,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:10,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:15,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:19,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:28:19,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:20,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:20,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:21,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:28:21,267 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:28:21,267 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:28:21,269 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:28:21,678 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:28:21,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:22,678 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:28:22,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:23,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:24,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:25,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:26,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:28,957 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:28:31,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:28:31,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:32,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:32,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:32,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:28:32,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:28:32,503 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:28:32,505 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:28:32,688 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:28:33,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:34,689 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:28:37,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:42,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:42,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:28:42,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:43,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:44,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:28:44,275 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:28:44,275 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:28:44,276 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:28:44,699 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:28:44,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:45,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:46,701 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:28:46,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:47,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:47,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:48,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:49,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:50,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:28:52,773 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:57,773 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:28:58,960 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:29:02,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:29:05,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:29:05,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:06,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:06,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:29:06,943 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:29:06,943 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:29:06,945 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:29:07,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:07,733 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:29:08,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:08,734 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:29:08,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:29:09,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:10,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:14,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:29:18,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:29:19,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:19,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:20,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:29:20,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:29:20,467 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:29:20,467 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:29:20,469 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:29:20,745 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:29:20,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:21,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:22,746 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:29:22,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:23,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:24,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:25,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:25,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:29:26,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:27,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:28,960 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:29:28,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:29,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:30,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:31,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:29:31,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:36,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:29:39,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:29:39,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:40,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:41,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:29:41,319 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:29:41,319 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:29:41,321 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:29:41,764 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:29:41,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:42,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:29:42,765 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:29:42,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:43,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:29:47,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:29:52,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:29:57,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:29:58,961 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:30:02,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:06,416 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:30:06,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:30:07,416 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:30:07,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:30:07,847 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:30:07,847 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:30:07,848 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:30:08,416 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:30:08,789 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:30:08,790 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:30:08,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:09,416 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:30:13,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:18,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:23,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:28,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:28,963 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:30:29,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:30:29,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:30:30,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:30:30,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:30:30,881 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:30:30,881 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:30:30,883 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:30:31,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:30:31,810 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:30:32,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:30:32,811 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:30:33,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:30:33,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:38,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:43,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:48,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:53,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:58,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:30:58,964 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:31:02,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:31:02,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:03,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:03,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:31:04,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:31:04,619 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:31:04,619 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:31:04,620 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:31:04,854 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:31:04,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:05,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:06,856 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:31:06,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:07,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:09,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:31:14,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:31:19,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:31:24,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:31:24,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:25,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:25,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:31:25,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:31:25,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:31:25,867 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:31:25,869 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:31:25,873 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:31:26,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:26,874 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:31:27,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:28,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:28,966 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:31:29,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:30,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:31,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:31,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:31:32,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:36,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:31:37,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:31:37,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:38,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:38,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:31:38,999 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:31:38,999 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:31:39,001 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:31:39,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:39,887 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:31:40,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:40,888 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:31:41,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:42,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:31:42,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:43,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:44,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:45,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:46,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:47,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:31:47,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:48,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:49,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:50,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:51,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:31:52,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:31:57,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:31:58,967 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:31:59,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:31:59,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:00,735 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:01,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:32:01,183 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:32:01,183 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:32:01,185 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:32:01,735 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:01,936 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:32:02,735 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:02,937 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:32:03,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:32:03,735 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:04,736 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:05,735 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:06,735 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:08,736 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:32:11,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:12,496 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:12,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:32:12,999 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:32:12,999 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:32:13,000 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:32:13,496 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:13,948 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:32:14,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:32:14,496 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:14,949 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:32:19,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:32:24,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:32:26,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:32:26,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:27,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:28,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:32:28,055 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:32:28,055 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:32:28,057 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:32:28,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:28,970 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:32:28,976 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:32:28,976 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:32:29,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:29,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:32:30,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:34,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:32:39,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:32:44,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:32:46,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:32:46,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:47,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:47,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:32:47,639 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:32:47,640 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:32:47,641 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:32:48,001 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:32:48,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:32:49,002 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:32:50,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:32:55,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:32:58,969 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:33:00,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:05,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:10,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:13,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:33:13,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:14,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:14,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:33:14,755 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:33:14,755 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:33:14,758 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:33:15,024 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:33:15,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:16,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:16,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:17,040 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:33:17,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:18,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:22,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:27,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:28,970 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:33:32,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:36,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:33:36,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:37,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:37,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:33:37,543 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:33:37,544 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:33:37,545 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:33:38,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:38,074 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:33:38,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:39,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:39,074 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:33:40,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:43,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:48,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:53,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:56,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:33:56,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:57,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:58,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:33:58,109 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:33:58,109 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:33:58,111 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:33:58,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:33:58,974 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:33:58,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:33:59,106 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:33:59,106 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:33:59,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:00,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:01,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:04,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:34:05,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:06,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:07,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:34:07,371 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:34:07,372 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:34:07,373 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:34:07,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:08,117 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:34:08,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:09,118 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:34:10,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:34:15,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:34:18,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:34:19,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:19,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:20,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:34:20,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:34:20,646 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:34:20,647 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:34:20,648 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:34:20,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:21,130 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:34:21,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:22,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:23,132 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:34:23,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:24,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:25,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:34:25,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:26,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:28,974 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:34:31,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:31,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:34:32,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:33,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:34:33,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:34:33,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:34:33,252 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:34:33,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:34,156 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:34:34,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:35,170 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:34:35,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:36,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:37,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:37,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:34:42,872 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:34:47,872 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:34:51,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:34:51,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:52,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:52,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:34:53,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:34:53,571 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:34:53,571 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:34:53,573 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:34:53,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:54,187 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:34:54,187 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:34:54,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:55,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:56,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:57,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:34:58,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:34:58,975 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:35:02,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:03,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:03,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:35:03,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:35:03,695 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:35:03,696 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:35:04,196 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:35:04,196 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:35:04,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:04,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:35:05,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:06,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:07,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:08,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:09,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:10,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:10,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:35:11,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:15,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:35:19,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:35:19,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:20,095 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:20,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:35:20,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:35:20,754 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:35:20,755 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:35:20,756 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:35:21,095 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:21,241 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:35:22,095 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:22,241 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:35:25,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:35:28,979 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:35:30,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:35:34,320 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:35:34,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:35,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:35,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:35:35,743 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:35:35,743 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:35:35,745 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:35:36,255 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:35:36,255 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:35:36,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:36,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:35:37,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:38,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:39,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:42,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:35:47,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:35:52,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:35:53,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:35:53,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:54,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:54,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:35:54,999 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:35:54,999 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:35:55,002 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:35:55,287 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:35:55,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:56,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:35:56,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:57,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:58,003 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:35:58,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:35:58,978 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:35:59,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:00,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:01,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:02,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:03,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:36:03,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:04,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:08,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:36:13,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:36:18,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:36:23,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:36:28,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:36:28,980 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:36:32,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:36:32,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:33,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:33,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:36:33,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:36:33,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:36:33,955 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:36:33,957 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:36:34,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:34,366 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:36:34,366 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:36:35,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:36,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:37,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:38,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:39,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:36:39,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:44,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:36:49,280 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:36:50,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:36:50,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:51,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:51,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:36:51,791 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:36:51,792 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:36:51,793 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:36:52,383 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:36:52,384 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:36:52,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:53,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:54,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:54,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:36:55,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:36:58,980 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:36:59,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:02,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:37:02,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:03,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:03,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:37:03,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:37:03,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:37:03,941 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:37:04,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:04,396 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:37:04,397 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:37:05,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:05,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:06,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:07,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:08,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:09,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:10,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:11,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:11,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:12,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:13,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:14,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:15,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:16,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:17,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:17,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:18,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:19,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:20,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:21,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:22,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:22,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:23,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:37:23,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:23,315 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:37:23,315 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:37:23,316 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:37:23,415 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:37:24,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:24,416 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:37:25,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:26,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:27,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:28,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:28,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:28,997 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:37:29,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:30,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:31,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:32,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:33,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:33,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:34,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:38,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:43,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:44,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:37:44,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:45,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:46,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:37:46,391 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:37:46,391 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:37:46,392 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:37:46,436 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:37:46,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:47,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:37:48,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:48,438 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:37:53,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:58,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:37:58,983 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:38:03,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:38:08,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:38:13,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:38:14,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:38:14,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:15,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:16,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:38:16,287 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:38:16,287 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:38:16,289 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:38:16,464 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:38:16,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:17,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:18,466 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:38:18,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:19,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:38:19,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:20,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:21,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:24,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:38:28,984 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:38:29,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:38:34,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:38:35,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:38:35,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:36,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:36,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:38:36,647 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:38:36,647 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:38:36,649 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:38:37,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:37,481 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:38:38,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:38,482 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:38:39,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:40,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:38:40,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:41,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:42,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:43,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:44,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:45,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:46,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:38:46,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:47,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:48,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:49,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:50,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:51,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:38:51,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:51,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:38:51,627 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:38:51,627 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:38:51,629 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:38:52,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:38:52,496 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:38:52,497 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:38:56,629 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:38:58,988 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:39:01,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:39:01,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:02,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:02,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:39:02,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:39:02,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:39:02,955 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:39:02,957 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:39:03,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:03,508 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:39:04,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:04,509 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:39:05,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:06,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:07,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:08,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:39:13,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:39:18,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:39:23,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:39:24,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:39:24,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:25,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:26,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:39:26,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:39:26,483 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:39:26,485 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:39:26,532 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:39:26,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:27,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:28,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:39:28,534 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:39:28,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:28,987 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:39:29,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:33,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:39:38,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:39:43,953 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:39:45,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:39:46,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:46,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:47,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:39:47,491 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:39:47,491 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:39:47,493 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:39:47,554 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:39:47,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:48,555 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:39:48,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:49,493 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:39:49,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:50,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:51,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:52,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:53,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:54,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:54,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:39:55,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:56,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:57,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:58,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:58,989 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:39:59,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:39:59,989 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:40:00,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:01,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:02,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:03,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:04,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:05,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:40:07,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:08,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:09,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:40:09,134 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:40:09,134 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:40:09,135 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:40:09,576 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:40:09,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:10,577 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:40:10,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:11,136 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:40:11,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:12,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:13,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:14,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:15,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:16,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:16,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:40:17,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:18,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:19,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:20,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:21,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:22,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:40:22,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:23,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:24,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:27,324 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:27,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:40:28,324 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:28,990 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:40:28,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:40:28,997 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:40:28,997 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:40:28,998 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:40:29,324 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:29,620 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:40:30,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:30,621 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:40:31,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:32,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:32,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:40:33,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:34,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:35,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:37,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:40:40,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:40:40,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:41,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:42,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:40:42,119 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:40:42,119 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:40:42,120 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:40:42,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:42,633 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:40:42,633 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:40:43,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:40:43,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:44,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:45,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:46,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:47,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:48,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:48,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:40:49,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:50,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:51,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:52,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:53,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:54,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:40:58,991 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:40:59,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:40:59,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:40:59,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:41:00,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:01,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:41:01,227 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:41:01,227 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:41:01,228 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:41:01,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:01,666 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:41:02,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:02,667 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:41:03,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:04,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:05,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:05,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:41:06,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:07,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:10,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:41:15,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:41:17,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:41:17,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:18,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:19,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:41:19,019 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:41:19,019 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:41:19,020 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:41:19,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:19,699 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:41:20,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:20,699 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:41:21,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:41:21,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:22,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:23,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:24,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:25,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:26,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:26,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:41:27,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:28,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:28,993 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:41:29,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:30,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:31,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:31,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:41:32,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:33,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:34,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:35,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:36,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:37,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:41:37,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:40,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:41,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:42,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:41:42,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:41:42,039 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:41:42,040 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:41:42,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:42,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:41:42,719 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:41:42,719 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:41:43,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:44,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:45,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:46,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:47,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:48,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:48,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:41:49,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:50,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:51,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:52,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:53,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:54,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:54,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:41:55,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:56,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:57,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:58,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:58,994 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:41:59,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:41:59,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:42:00,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:01,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:02,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:42:02,263 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:42:02,263 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:42:02,265 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:42:02,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:02,737 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:42:03,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:04,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:04,739 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:42:05,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:05,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:42:06,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:07,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:08,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:09,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:10,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:11,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:11,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:42:12,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:17,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:42:22,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:42:22,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:23,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:23,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:42:23,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:42:23,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:42:23,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:42:23,796 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:42:24,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:24,762 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:42:24,763 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:42:28,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:42:28,997 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:42:33,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:42:38,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:42:40,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:42:40,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:41,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:42,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:42:42,315 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:42:42,315 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:42:42,316 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:42:42,779 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:42:42,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:43,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:44,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:42:44,781 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:42:44,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:45,852 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:46,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:47,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:48,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:49,852 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:42:51,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:52,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:53,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:42:53,023 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:42:53,023 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:42:53,025 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:42:53,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:53,791 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:42:54,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:54,792 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:42:55,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:42:55,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:56,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:57,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:58,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:42:58,997 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:42:59,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:00,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:00,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:01,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:02,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:03,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:04,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:05,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:06,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:06,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:07,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:08,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:09,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:10,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:11,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:11,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:12,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:13,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:43:13,346 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:43:13,346 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:43:13,348 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:43:13,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:13,807 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:43:14,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:14,808 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:43:17,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:22,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:27,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:29,000 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:43:32,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:43:32,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:32,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:33,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:34,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:43:34,135 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:43:34,135 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:43:34,137 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:43:34,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:34,828 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:43:34,828 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:43:35,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:36,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:37,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:43:38,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:43,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:48,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:53,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:58,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:43:59,017 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:44:04,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:44:06,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:44:06,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:07,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:07,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:44:07,883 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:44:07,883 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:44:07,884 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:44:08,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:08,853 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:44:08,854 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:44:09,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:09,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:44:10,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:11,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:12,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:15,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:44:20,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:44:25,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:44:29,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:44:31,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:44:35,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:44:35,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:36,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:36,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:44:37,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:44:37,291 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:44:37,291 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:44:37,293 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:44:37,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:37,885 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:44:38,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:38,886 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:44:39,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:40,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:41,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:42,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:42,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:44:43,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:46,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:47,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:48,016 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:44:48,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:44:48,059 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:44:48,059 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:44:48,061 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:44:48,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:48,897 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:44:48,897 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:44:49,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:44:53,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:44:58,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:44:59,004 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:44:59,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:44:59,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:00,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:01,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:45:01,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:45:01,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:45:01,341 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:45:01,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:01,908 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:45:02,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:02,909 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:45:03,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:45:03,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:08,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:45:13,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:45:18,343 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:45:23,343 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:45:24,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:45:24,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:25,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:25,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:45:25,995 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:45:25,995 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:45:25,998 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:45:26,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:26,945 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:45:26,945 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:45:27,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:28,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:28,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:45:29,008 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:45:29,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:30,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:31,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:33,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:34,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:45:34,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:35,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:45:35,391 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:45:35,391 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:45:35,392 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:45:35,954 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:45:35,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:36,955 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:45:36,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:37,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:38,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:39,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:40,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:45:40,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:45,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:45:50,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:45:54,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:45:54,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:55,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:55,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:45:55,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:45:55,671 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:45:55,671 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:45:55,673 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:45:55,973 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:45:56,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:45:56,974 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:45:59,007 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:46:01,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:06,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:11,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:16,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:21,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:21,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:46:21,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:22,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:23,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:46:23,323 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:46:23,323 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:46:23,325 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:46:23,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:24,012 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:46:24,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:25,013 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:46:25,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:26,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:26,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:27,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:28,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:29,008 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:46:29,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:30,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:31,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:31,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:36,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:39,280 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:46:39,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:40,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:40,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:46:40,588 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:46:40,588 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:46:40,589 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:46:41,047 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:46:41,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:42,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:42,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:43,049 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:46:43,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:44,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:48,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:53,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:58,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:46:58,412 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:46:58,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:59,009 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:46:59,412 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:46:59,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:46:59,827 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:46:59,827 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:46:59,829 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:47:00,087 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:47:00,412 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:01,088 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:47:01,412 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:02,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:03,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:47:03,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:04,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:05,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:06,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:07,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:08,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:47:13,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:47:14,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:47:14,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:15,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:15,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:47:15,683 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:47:15,683 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:47:15,685 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:47:16,100 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:47:16,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:17,101 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:47:18,685 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:47:23,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:47:28,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:47:29,022 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:47:34,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:47:37,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:47:37,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:38,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:39,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:47:39,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:47:39,315 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:47:39,315 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:47:39,332 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:47:39,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:40,128 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:47:40,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:41,128 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:47:41,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:42,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:43,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:47:44,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:47:49,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:47:54,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:47:59,012 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:48:00,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:02,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:48:02,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:03,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:03,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:48:03,735 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:48:03,735 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:48:03,738 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:48:04,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:04,170 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:48:05,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:05,171 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:48:05,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:06,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:07,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:08,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:09,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:10,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:11,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:16,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:18,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:48:18,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:19,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:20,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:48:20,131 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:48:20,131 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:48:20,133 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:48:20,185 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:48:20,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:21,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:21,186 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:48:21,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:22,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:26,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:29,013 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:48:32,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:32,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:48:32,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:33,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:34,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:48:34,119 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:48:34,119 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:48:34,120 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:48:34,196 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:48:34,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:35,197 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:48:35,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:36,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:37,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:37,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:38,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:48:42,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:47,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:52,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:57,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:48:59,032 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:49:03,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:49:08,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:49:11,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:49:11,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:12,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:13,204 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:49:13,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:49:13,247 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:49:13,247 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:49:13,249 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:49:13,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:14,240 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:49:14,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:15,241 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:49:15,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:16,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:17,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:18,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:18,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:49:19,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:21,416 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:22,416 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:22,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:49:22,819 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:49:22,819 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:49:22,821 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:49:23,250 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:49:23,416 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:23,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:49:25,252 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:49:28,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:49:29,016 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:49:34,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:49:39,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:49:44,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:49:48,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:49:48,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:49,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:49,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:49:50,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:49:50,347 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:49:50,347 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:49:50,349 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:49:50,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:51,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:49:51,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:52,289 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:49:52,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:49:55,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:49:59,018 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:50:01,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:06,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:08,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:50:08,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:09,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:10,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:50:10,038 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:50:10,038 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:50:10,039 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:50:10,327 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:50:10,327 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:50:10,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:11,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:11,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:12,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:13,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:14,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:16,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:21,396 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:26,396 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:26,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:50:26,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:27,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:28,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:50:28,543 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:50:28,543 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:50:28,545 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:50:28,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:29,020 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:50:29,345 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:50:29,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:30,345 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:50:32,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:37,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:42,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:43,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:50:43,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:44,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:44,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:50:44,763 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:50:44,763 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:50:44,765 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:50:45,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:45,384 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:50:46,385 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:50:47,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:52,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:55,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:50:55,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:56,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:57,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:50:57,259 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:50:57,259 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:50:57,271 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:50:57,396 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:50:57,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:50:58,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:50:58,397 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:50:59,020 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:51:04,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:51:09,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:51:14,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:51:19,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:51:24,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:51:25,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:51:25,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:26,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:26,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:51:26,903 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:51:26,903 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:51:26,905 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:51:27,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:27,420 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:51:28,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:28,421 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:51:29,022 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:51:29,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:30,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:51:35,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:51:40,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:51:45,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:51:48,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:51:48,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:49,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:50,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:51:50,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:51:50,471 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:51:50,471 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:51:50,473 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:51:50,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:51,441 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:51:51,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:52,442 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:51:52,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:53,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:54,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:51:55,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:51:59,023 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:52:01,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:52:03,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:52:03,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:04,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:05,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:52:05,075 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:52:05,075 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:52:05,077 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:52:05,454 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:52:05,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:06,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:52:06,455 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:52:06,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:07,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:08,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:09,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:10,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:11,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:52:11,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:12,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:16,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:52:21,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:52:21,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:22,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:22,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:52:23,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:52:23,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:52:23,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:52:23,253 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:52:23,470 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:52:23,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:24,471 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:52:24,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:25,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:28,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:52:29,026 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:52:34,027 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:52:35,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:52:35,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:36,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:37,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:52:37,191 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:52:37,191 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:52:37,193 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:52:37,483 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:52:37,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:38,484 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:52:38,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:39,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:52:39,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:40,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:41,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:42,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:43,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:44,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:44,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:52:48,616 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:49,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:49,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:52:50,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:52:50,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:52:50,039 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:52:50,040 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:52:50,512 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:52:50,512 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:52:50,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:51,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:52,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:52:55,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:52:59,026 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:53:01,027 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:06,027 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:11,028 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:12,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:53:12,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:13,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:14,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:53:14,534 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:53:14,534 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:53:14,536 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:53:14,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:15,533 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:53:15,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:16,533 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:53:16,537 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:16,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:17,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:21,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:26,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:29,027 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:53:32,028 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:37,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:41,216 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:53:41,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:42,216 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:42,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:42,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:53:42,759 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:53:42,759 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:53:42,760 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:53:43,216 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:43,558 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:53:44,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:44,559 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:53:45,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:46,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:47,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:48,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:48,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:49,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:52,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:53,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:53,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:54,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:53:54,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:53:54,003 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:53:54,005 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:53:54,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:54,584 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:53:54,584 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:53:55,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:56,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:53:59,005 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:53:59,030 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:54:01,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:02,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:03,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:54:03,183 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:54:03,183 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:54:03,184 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:54:03,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:03,593 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:54:04,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:54:04,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:04,594 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:54:05,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:09,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:54:13,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:54:13,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:14,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:14,835 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:54:15,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:54:15,387 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:54:15,387 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:54:15,389 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:54:15,604 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:54:15,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:16,605 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:54:20,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:54:25,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:54:29,030 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:54:31,031 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:54:36,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:54:40,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:54:40,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:41,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:41,431 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:54:41,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:54:41,679 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:54:41,679 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:54:41,680 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:54:42,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:42,629 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:54:42,629 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:54:43,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:44,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:45,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:54:47,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:54:52,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:54:57,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:54:59,031 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:54:59,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:54:59,230 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:00,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:00,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:55:00,691 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:55:00,691 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:55:00,692 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:55:01,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:01,676 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:55:02,676 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:55:02,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:55:07,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:55:12,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:55:15,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:55:15,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:16,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:16,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:55:16,919 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:55:16,919 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:55:16,920 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:55:17,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:17,689 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:55:17,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:55:18,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:18,690 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:55:19,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:20,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:21,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:22,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:23,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:23,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:55:24,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:25,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:26,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:27,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:28,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:29,034 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:55:29,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:55:29,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:30,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:31,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:32,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:33,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:34,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:55:34,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:35,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:36,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:37,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:55:37,127 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:55:37,127 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:55:37,129 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:55:37,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:37,722 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:55:38,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:38,723 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:55:39,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:40,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:55:45,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:55:46,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:55:46,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:47,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:47,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:55:47,783 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:55:47,783 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:55:47,785 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:55:48,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:55:48,734 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:55:48,734 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:55:50,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:55:55,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:55:59,034 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:56:01,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:56:01,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:56:01,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:02,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:02,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:56:02,820 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:56:02,820 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:56:02,822 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:56:03,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:03,749 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:56:04,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:04,750 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:56:05,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:06,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:56:11,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:56:16,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:56:21,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:56:26,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:56:26,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:27,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:27,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:56:28,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:56:28,167 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:56:28,167 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:56:28,169 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:56:28,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:28,769 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:56:28,770 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:56:29,035 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:56:29,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:30,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:31,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:32,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:33,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:33,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:56:34,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:35,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:36,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:37,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:38,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:38,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:56:39,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:40,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:41,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:42,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:43,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:44,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:56:44,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:45,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:46,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:47,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:48,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:49,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:56:49,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:50,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:51,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:52,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:53,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:56:53,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:56:53,459 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:56:53,461 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:56:53,797 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:56:53,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:54,798 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:56:54,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:55,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:56:55,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:56,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:57,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:58,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:56:59,037 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:56:59,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:00,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:01,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:01,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:02,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:03,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:04,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:05,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:57:05,591 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:57:05,591 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:57:05,592 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:57:05,808 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:57:05,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:06,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:06,809 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:57:11,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:16,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:21,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:26,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:27,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:57:27,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:28,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:28,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:57:28,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:57:28,611 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:57:28,613 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:57:28,841 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:57:29,038 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:57:29,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:30,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:30,843 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:57:31,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:32,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:32,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:33,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:34,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:37,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:40,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:57:40,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:41,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:42,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:57:42,031 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:57:42,031 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:57:42,032 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:57:42,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:42,853 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:57:42,853 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:57:43,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:43,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:44,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:57:48,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:53,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:58,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:57:59,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:58:03,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:58:06,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:58:06,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:07,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:08,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:58:08,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:58:08,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:58:08,483 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:58:08,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:08,485 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:58:08,890 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:58:10,892 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:58:13,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:58:18,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:58:23,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:58:28,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:58:29,041 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:58:34,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:58:36,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:58:36,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:37,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:38,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:58:38,247 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:58:38,247 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:58:38,249 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:58:38,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:38,925 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:58:38,925 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:58:39,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:58:39,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:44,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:58:47,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:58:47,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:48,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:48,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:58:48,855 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:58:48,855 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:58:48,857 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:58:48,935 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:58:49,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:49,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:58:50,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:50,937 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:58:51,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:52,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:58:55,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:58:59,044 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:59:01,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:06,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:11,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:16,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:21,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:26,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:26,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:59:26,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:59:27,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:59:27,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:59:27,691 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:59:27,691 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:59:27,693 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:59:28,005 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:59:28,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:59:29,006 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:59:29,043 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 10:59:29,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:59:30,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:59:31,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:31,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:59:32,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:59:36,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:41,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:45,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 10:59:45,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:59:46,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:59:46,767 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:47,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 10:59:47,279 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 10:59:47,279 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 10:59:47,281 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 10:59:47,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:59:48,022 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 10:59:48,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 10:59:49,023 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 10:59:52,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:57,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 10:59:59,044 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:00:01,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:00:01,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:02,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:02,291 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:00:02,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:00:02,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:00:02,623 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:00:02,625 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:00:03,036 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:00:03,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:04,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:05,038 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:00:05,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:06,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:07,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:08,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:08,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:00:12,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:13,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:13,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:00:13,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:00:13,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:00:13,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:00:13,940 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:00:14,062 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:00:14,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:15,063 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:00:15,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:16,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:17,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:18,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:18,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:00:19,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:20,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:21,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:22,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:23,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:24,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:24,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:00:25,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:26,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:27,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:28,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:29,048 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:00:29,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:30,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:00:30,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:31,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:32,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:33,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:34,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:35,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:00:35,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:36,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:37,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:00:37,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:37,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:00:37,503 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:00:37,504 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:00:38,084 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:00:38,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:39,085 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:00:39,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:40,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:40,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:00:41,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:42,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:43,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:44,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:45,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:46,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:00:46,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:51,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:00:56,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:00:58,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:00:58,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:59,047 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:00:59,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:00:59,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:00:59,879 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:00:59,879 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:00:59,881 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:01:00,103 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:01:00,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:01,104 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:01:01,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:06,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:11,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:16,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:17,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:01:17,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:18,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:19,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:01:19,051 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:01:19,051 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:01:19,052 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:01:19,121 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:01:19,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:21,123 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:01:22,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:26,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:01:27,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:27,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:28,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:28,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:01:28,743 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:01:28,743 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:01:28,744 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:01:28,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:29,049 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:01:29,158 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:01:29,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:30,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:31,160 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:01:31,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:32,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:33,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:33,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:34,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:35,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:38,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:43,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:48,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:52,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:01:52,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:53,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:54,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:54,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:01:54,043 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:01:54,043 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:01:54,045 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:01:54,181 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:01:54,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:55,182 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:01:55,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:56,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:57,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:58,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:01:59,050 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:01:59,051 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:01:59,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:00,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:01,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:02,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:03,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:04,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:02:09,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:02:12,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:02:12,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:13,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:14,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:02:14,355 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:02:14,355 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:02:14,357 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:02:14,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:15,201 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:02:15,357 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:02:15,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:16,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:17,202 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:02:17,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:18,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:20,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:02:25,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:02:29,053 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:02:30,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:02:30,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:30,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:02:31,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:32,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:02:32,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:02:32,611 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:02:32,612 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:02:32,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:33,216 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:02:33,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:34,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:35,250 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:02:35,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:36,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:02:41,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:02:43,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:02:44,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:44,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:45,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:02:45,361 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:02:45,362 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:02:45,363 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:02:45,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:46,260 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:02:46,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:47,261 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:02:47,363 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:02:47,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:48,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:49,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:50,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:51,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:52,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:52,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:02:53,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:02:58,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:02:59,052 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:03:04,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:03:05,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:03:05,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:06,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:07,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:03:07,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:03:07,587 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:03:07,589 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:03:07,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:08,293 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:03:08,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:09,294 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:03:09,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:03:09,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:10,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:11,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:12,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:13,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:14,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:14,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:03:15,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:16,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:17,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:18,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:19,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:03:24,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:03:24,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:25,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:25,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:03:26,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:03:26,319 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:03:26,319 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:03:26,320 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:03:26,323 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:03:26,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:27,324 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:03:27,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:28,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:29,054 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:03:29,326 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:03:29,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:30,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:31,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:03:31,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:36,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:03:41,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:03:45,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:03:46,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:46,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:47,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:03:47,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:03:47,443 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:03:47,443 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:03:47,444 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:03:47,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:48,387 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:03:48,387 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:03:48,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:03:52,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:03:57,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:03:59,057 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:04:03,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:04:03,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:04:03,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:04,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:04,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:04:04,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:04:04,459 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:04:04,460 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:04:05,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:05,404 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:04:06,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:06,405 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:04:07,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:08,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:04:08,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:09,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:10,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:11,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:12,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:13,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:14,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:04:14,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:15,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:16,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:17,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:17,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:04:17,167 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:04:17,168 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:04:17,169 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:04:17,417 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:04:18,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:18,418 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:04:19,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:04:19,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:20,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:24,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:04:29,057 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:04:30,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:04:35,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:04:40,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:04:41,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:04:41,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:42,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:42,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:04:42,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:04:42,559 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:04:42,561 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:04:43,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:43,445 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:04:44,446 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:04:45,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:04:50,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:04:50,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:51,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:51,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:04:52,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:04:52,171 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:04:52,171 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:04:52,172 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:04:52,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:04:52,456 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:04:52,456 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:04:57,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:04:59,058 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:05:03,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:05:08,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:05:13,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:05:18,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:05:18,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:05:19,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:19,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:20,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:05:20,427 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:05:20,427 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:05:20,428 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:05:20,505 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:05:20,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:21,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:22,507 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:05:22,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:23,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:05:23,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:24,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:25,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:26,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:27,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:28,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:05:28,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:29,076 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:05:29,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:30,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:31,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:32,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:33,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:34,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:05:34,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:35,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:36,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:37,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:38,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:39,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:05:39,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:40,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:05:40,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:05:40,559 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:05:40,560 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:05:40,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:41,522 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:05:42,523 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:05:45,561 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:05:50,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:05:55,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:05:56,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:05:56,230 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:57,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:57,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:05:57,510 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:05:57,511 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:05:57,512 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:05:57,549 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:05:58,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:05:58,550 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:05:59,060 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:05:59,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:00,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:01,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:01,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:02,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:03,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:04,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:05,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:06,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:06,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:07,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:08,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:09,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:10,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:11,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:11,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:12,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:13,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:16,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:18,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:06:18,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:19,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:19,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:06:19,923 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:06:19,923 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:06:19,925 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:06:20,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:20,571 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:06:20,571 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:06:21,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:26,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:29,062 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:06:32,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:36,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:06:36,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:37,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:37,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:38,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:06:38,067 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:06:38,067 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:06:38,069 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:06:38,588 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:06:38,588 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:06:38,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:39,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:43,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:48,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:53,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:54,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:06:54,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:55,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:56,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:06:56,087 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:06:56,088 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:06:56,104 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:06:56,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:56,616 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:06:56,616 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:06:57,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:06:58,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:06:59,063 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:07:04,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:07:09,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:07:11,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:07:11,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:12,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:13,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:07:13,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:07:13,175 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:07:13,177 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:07:13,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:13,632 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:07:14,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:07:14,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:14,633 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:07:15,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:19,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:07:24,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:07:25,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:07:25,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:26,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:27,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:07:27,099 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:07:27,099 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:07:27,101 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:07:27,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:27,646 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:07:28,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:28,647 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:07:29,064 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:07:29,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:30,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:07:35,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:07:36,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:07:36,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:37,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:38,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:07:38,011 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:07:38,011 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:07:38,013 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:07:38,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:38,658 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:07:38,658 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:07:41,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:07:46,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:07:47,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:07:47,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:48,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:48,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:07:48,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:07:48,747 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:07:48,749 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:07:49,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:49,668 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:07:50,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:50,669 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:07:51,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:07:51,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:07:56,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:07:59,066 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:08:02,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:08:02,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:08:02,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:03,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:04,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:08:04,179 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:08:04,179 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:08:04,181 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:08:04,679 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:08:04,679 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:08:04,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:05,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:06,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:07,181 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:08:07,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:08,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:09,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:10,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:12,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:08:17,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:08:19,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:08:19,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:20,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:20,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:08:20,863 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:08:20,863 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:08:20,865 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:08:21,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:21,713 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:08:22,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:22,714 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:08:22,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:08:23,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:24,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:25,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:26,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:27,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:28,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:28,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:08:29,067 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:08:34,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:08:39,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:08:44,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:08:49,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:08:53,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:08:53,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:54,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:54,435 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:08:54,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:08:54,712 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:08:54,713 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:08:54,714 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:08:54,760 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:08:55,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:56,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:56,762 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:08:57,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:58,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:08:59,070 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:08:59,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:00,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:00,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:05,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:05,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:09:05,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:06,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:07,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:09:07,267 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:09:07,267 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:09:07,269 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:09:07,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:07,771 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:09:08,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:08,772 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:09:10,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:14,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:09:14,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:15,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:15,435 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:15,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:09:15,731 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:09:15,731 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:09:15,746 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:09:15,779 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:09:16,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:16,780 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:09:17,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:18,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:19,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:20,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:21,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:21,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:22,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:23,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:24,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:27,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:29,069 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:09:33,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:37,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:09:37,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:38,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:38,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:39,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:09:39,063 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:09:39,063 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:09:39,065 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:09:39,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:39,800 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:09:40,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:40,801 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:09:41,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:09:44,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:49,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:54,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:59,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:09:59,071 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:09:59,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:09:59,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:00,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:00,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:10:00,799 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:10:00,799 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:10:00,801 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:10:00,818 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:10:01,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:02,257 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:02,820 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:10:04,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:10:09,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:10:14,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:10:19,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:10:24,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:10:29,074 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:10:30,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:10:32,431 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:10:32,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:33,431 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:33,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:10:33,847 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:10:33,847 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:10:33,848 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:10:33,872 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:10:34,431 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:34,874 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:10:35,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:10:40,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:10:45,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:10:50,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:10:51,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:10:51,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:52,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:52,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:10:52,743 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:10:52,743 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:10:52,744 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:10:52,888 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:10:53,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:54,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:54,890 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:10:55,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:56,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:10:56,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:57,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:58,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:10:59,073 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:10:59,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:00,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:02,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:11:07,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:11:11,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:11:11,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:12,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:12,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:11:12,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:11:12,895 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:11:12,895 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:11:12,896 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:11:12,905 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:11:13,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:14,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:14,907 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:11:15,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:16,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:18,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:11:23,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:11:28,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:11:29,075 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:11:34,076 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:11:34,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:11:34,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:35,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:36,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:11:36,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:11:36,623 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:11:36,625 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:11:36,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:36,940 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:11:37,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:38,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:38,946 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:11:39,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:11:39,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:40,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:41,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:42,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:43,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:44,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:11:44,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:45,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:46,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:47,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:48,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:49,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:50,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:11:50,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:51,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:52,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:53,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:54,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:55,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:56,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:11:56,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:57,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:58,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:11:59,076 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:11:59,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:00,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:01,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:12:01,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:02,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:03,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:04,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:05,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:06,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:12:06,579 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:12:06,579 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:12:06,580 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:12:06,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:06,984 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:12:07,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:12:07,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:08,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:08,986 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:12:09,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:10,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:11,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:12,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:12:17,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:12:22,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:12:25,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:12:25,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:26,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:26,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:12:26,615 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:12:26,615 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:12:26,616 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:12:26,999 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:12:27,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:28,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:28,617 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:12:29,001 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:12:29,077 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:12:29,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:34,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:12:39,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:12:39,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:12:39,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:40,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:40,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:12:40,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:12:40,623 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:12:40,625 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:12:41,028 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:12:41,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:42,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:43,030 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:12:43,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:44,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:12:44,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:45,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:12:49,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:12:54,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:12:59,078 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:13:00,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:05,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:07,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:13:07,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:08,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:08,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:13:08,591 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:13:08,591 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:13:08,592 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:13:09,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:09,065 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:13:10,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:10,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:11,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:11,067 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:13:15,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:20,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:25,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:29,080 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:13:31,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:36,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:40,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:13:40,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:41,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:41,767 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:42,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:13:42,183 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:13:42,183 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:13:42,184 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:13:42,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:43,099 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:13:43,099 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:13:43,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:47,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:52,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:13:52,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:53,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:53,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:53,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:13:53,475 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:13:53,475 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:13:53,476 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:13:54,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:54,108 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:13:55,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:55,110 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:13:56,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:57,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:58,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:13:59,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:13:59,082 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:14:04,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:14:09,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:14:12,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:14:12,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:13,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:13,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:14:13,435 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:14:13,435 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:14:13,436 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:14:14,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:14,125 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:14:14,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:14:15,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:15,126 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:14:16,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:17,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:18,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:19,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:20,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:14:20,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:21,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:22,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:23,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:24,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:25,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:26,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:14:26,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:27,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:28,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:29,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:29,082 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:14:30,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:31,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:31,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:14:32,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:33,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:34,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:35,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:36,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:37,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:14:37,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:38,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:39,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:40,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:41,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:42,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:43,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:14:43,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:44,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:45,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:46,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:47,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:48,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:14:48,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:49,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:50,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:51,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:52,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:52,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:14:52,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:14:52,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:14:52,253 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:14:53,155 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:14:53,156 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:14:53,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:53,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:14:54,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:55,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:56,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:57,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:58,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:14:59,083 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:14:59,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:14:59,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:00,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:01,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:02,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:03,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:04,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:15:04,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:09,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:15:14,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:15:14,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:15:14,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:15,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:15,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:15:15,671 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:15:15,671 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:15:15,673 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:15:16,192 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:15:16,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:17,193 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:15:17,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:19,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:15:24,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:15:29,087 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:15:30,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:15:32,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:15:32,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:33,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:34,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:15:34,163 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:15:34,163 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:15:34,164 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:15:34,209 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:15:34,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:35,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:15:35,210 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:15:35,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:36,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:37,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:38,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:39,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:40,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:15:40,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:41,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:42,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:43,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:46,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:15:51,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:15:52,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:15:52,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:53,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:54,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:15:54,499 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:15:54,499 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:15:54,500 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:15:54,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:15:55,226 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:15:57,228 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:15:57,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:15:59,086 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:16:03,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:16:08,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:16:12,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:16:12,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:13,197 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:13,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:16:13,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:16:13,576 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:16:13,576 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:16:13,578 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:16:14,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:14,242 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:16:15,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:15,244 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:16:16,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:17,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:18,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:19,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:19,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:16:20,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:21,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:22,628 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:23,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:24,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:16:24,015 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:16:24,015 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:16:24,016 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:16:24,253 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:16:24,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:25,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:16:25,254 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:16:25,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:29,087 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:16:30,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:16:35,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:16:40,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:16:44,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:16:44,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:45,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:45,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:16:45,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:16:45,999 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:16:45,999 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:16:46,001 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:16:46,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:46,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:16:47,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:47,289 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:16:48,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:49,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:50,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:51,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:51,227 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:16:52,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:53,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:16:56,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:16:59,089 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:17:02,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:03,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:17:03,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:17:04,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:17:05,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:17:05,268 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:17:05,269 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:17:05,270 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:17:05,304 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:17:05,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:17:07,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:07,306 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:17:12,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:17,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:18,773 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:17:18,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:17:19,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:17:20,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:17:20,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:17:20,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:17:20,368 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:17:20,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:17:21,318 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:17:21,318 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:17:21,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:17:22,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:27,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:29,090 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:17:33,091 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:38,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:38,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:17:38,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:17:39,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:17:40,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:17:40,347 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:17:40,347 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:17:40,348 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:17:40,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:17:41,334 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:17:41,335 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:17:41,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:17:43,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:48,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:53,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:58,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:17:59,091 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:18:04,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:18:09,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:18:14,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:18:17,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:18:17,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:18,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:19,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:18:19,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:18:19,295 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:18:19,295 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:18:19,296 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:18:19,368 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:18:19,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:20,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:21,370 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:18:21,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:22,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:23,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:24,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:24,596 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:18:25,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:26,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:27,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:28,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:29,093 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:18:29,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:30,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:18:35,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:18:35,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:18:35,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:36,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:36,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:18:36,699 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:18:36,699 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:18:36,701 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:18:37,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:37,383 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:18:38,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:39,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:39,386 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:18:40,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:18:40,332 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:18:45,332 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:18:50,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:18:55,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:18:59,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:19:01,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:06,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:08,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:19:08,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:09,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:10,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:19:10,291 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:19:10,291 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:19:10,292 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:19:10,459 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:19:10,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:11,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:11,460 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:19:11,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:12,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:13,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:14,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:15,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:16,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:16,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:17,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:21,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:26,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:29,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:19:32,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:37,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:37,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:19:37,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:38,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:39,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:19:39,151 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:19:39,151 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:19:39,153 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:19:39,500 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:19:39,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:40,501 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:19:40,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:41,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:42,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:42,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:47,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:52,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:55,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:19:55,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:56,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:56,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:19:56,679 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:19:56,679 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:19:56,681 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:19:57,291 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:57,518 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:19:57,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:19:58,291 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:19:58,519 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:19:59,096 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:20:03,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:20:08,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:20:13,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:20:14,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:20:14,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:15,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:15,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:20:15,471 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:20:15,471 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:20:15,486 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:20:15,533 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:20:16,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:16,534 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:20:17,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:18,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:18,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:20:19,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:20,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:21,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:22,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:23,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:24,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:20:24,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:25,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:26,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:29,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:20:29,113 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:20:34,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:20:39,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:20:44,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:20:49,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:20:52,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:20:52,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:53,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:54,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:20:54,063 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:20:54,063 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:20:54,065 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:20:54,566 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:20:54,566 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:20:54,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:55,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:20:55,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:20:59,099 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:21:00,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:05,100 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:10,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:15,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:15,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:21:15,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:21:16,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:21:17,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:21:17,323 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:21:17,323 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:21:17,325 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:21:17,602 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:21:17,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:21:18,603 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:21:18,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:21:19,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:21:20,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:20,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:21:21,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:21:25,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:29,100 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:21:31,101 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:35,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:21:35,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:21:36,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:21:36,539 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:37,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:21:37,171 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:21:37,171 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:21:37,172 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:21:37,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:21:37,620 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:21:38,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:21:38,621 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:21:42,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:47,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:52,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:57,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:21:59,101 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:22:03,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:22:07,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:22:07,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:08,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:08,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:22:08,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:22:08,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:22:08,747 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:22:08,748 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:22:09,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:09,669 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:22:10,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:10,670 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:22:11,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:12,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:13,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:14,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:22:14,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:15,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:16,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:19,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:22:24,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:22:29,105 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:22:30,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:22:35,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:22:35,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:22:35,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:36,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:37,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:22:37,067 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:22:37,067 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:22:37,069 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:22:37,708 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:22:37,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:38,709 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:22:38,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:39,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:40,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:22:40,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:41,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:42,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:45,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:22:46,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:47,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:48,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:22:48,887 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:48,887 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:22:48,888 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:22:48,889 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:22:49,730 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:22:49,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:50,731 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:22:50,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:50,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:22:51,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:52,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:53,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:54,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:22:56,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:22:59,104 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:23:02,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:23:06,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:23:06,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:07,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:07,619 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:23:08,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:23:08,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:23:08,235 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:23:08,238 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:23:08,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:08,775 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:23:08,776 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:23:09,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:10,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:11,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:12,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:13,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:23:13,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:14,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:18,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:23:23,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:23:28,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:23:28,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:29,105 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:23:29,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:23:29,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:29,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:23:29,755 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:23:29,755 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:23:29,756 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:23:29,814 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:23:30,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:30,815 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:23:34,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:23:39,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:23:42,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:23:42,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:43,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:44,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:23:44,018 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:23:44,018 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:23:44,020 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:23:44,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:44,829 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:23:44,829 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:23:45,020 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:23:50,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:23:55,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:23:58,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:23:58,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:23:59,109 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:23:59,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:00,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:24:00,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:24:00,067 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:24:00,067 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:24:00,069 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:24:00,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:00,845 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:24:00,845 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:24:01,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:02,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:03,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:05,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:24:09,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:24:09,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:10,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:10,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:24:10,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:24:10,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:24:10,611 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:24:10,612 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:24:10,867 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:24:11,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:12,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:12,869 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:24:13,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:14,212 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:16,212 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:24:21,213 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:24:24,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:24:24,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:25,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:26,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:24:26,035 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:24:26,035 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:24:26,037 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:24:26,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:26,884 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:24:26,884 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:24:27,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:24:27,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:28,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:29,108 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:24:29,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:30,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:31,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:32,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:32,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:24:33,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:34,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:35,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:38,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:24:40,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:41,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:41,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:24:41,519 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:24:41,519 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:24:41,520 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:24:41,905 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:24:42,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:42,906 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:24:43,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:43,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:24:44,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:45,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:46,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:47,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:48,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:49,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:49,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:24:50,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:51,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:52,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:53,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:54,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:55,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:24:55,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:56,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:24:59,109 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:25:00,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:02,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:25:02,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:03,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:04,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:25:04,119 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:25:04,119 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:25:04,121 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:25:04,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:04,926 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:25:04,926 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:25:05,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:05,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:06,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:07,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:08,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:09,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:10,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:10,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:11,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:12,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:13,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:14,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:15,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:16,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:16,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:19,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:20,969 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:21,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:25:21,447 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:25:21,447 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:25:21,448 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:25:21,942 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:25:21,969 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:22,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:22,943 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:25:27,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:29,112 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:25:33,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:38,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:40,660 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:25:40,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:41,660 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:42,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:25:42,139 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:25:42,139 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:25:42,140 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:25:42,660 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:42,980 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:25:42,980 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:25:43,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:43,660 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:25:48,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:53,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:58,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:25:59,112 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:25:59,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:25:59,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:00,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:01,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:26:01,071 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:26:01,071 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:26:01,072 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:26:01,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:01,995 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:26:02,996 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:26:04,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:26:09,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:26:14,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:26:19,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:26:24,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:26:29,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:26:29,113 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:26:33,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:26:33,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:34,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:34,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:26:35,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:26:35,239 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:26:35,239 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:26:35,254 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:26:35,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:36,039 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:26:36,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:37,040 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:26:37,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:40,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:26:45,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:26:49,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:26:49,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:50,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:50,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:26:50,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:26:50,568 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:26:50,568 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:26:50,569 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:26:51,067 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:26:51,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:52,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:26:53,069 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:26:55,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:26:59,114 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:27:01,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:02,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:27:02,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:03,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:03,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:27:03,875 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:27:03,875 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:27:03,877 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:27:04,109 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:27:04,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:05,110 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:27:05,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:06,488 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:06,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:07,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:08,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:09,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:12,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:17,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:22,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:27,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:29,133 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:27:31,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:27:31,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:32,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:32,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:32,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:27:32,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:27:32,955 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:27:32,956 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:27:33,161 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:27:33,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:34,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:35,163 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:27:35,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:36,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:38,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:43,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:46,168 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:27:46,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:47,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:47,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:27:47,471 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:27:47,471 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:27:47,472 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:27:48,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:48,175 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:27:48,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:49,176 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:27:53,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:56,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:27:56,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:57,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:57,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:27:57,983 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:27:57,983 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:27:57,984 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:27:58,184 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:27:58,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:27:58,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:27:59,117 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:27:59,185 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:27:59,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:28:00,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:28:04,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:28:09,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:28:13,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:28:13,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:28:14,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:28:14,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:28:14,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:28:14,687 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:28:14,687 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:28:14,689 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:28:15,206 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:28:15,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:28:17,208 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:28:19,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:28:24,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:28:29,118 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:28:30,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:28:35,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:28:36,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:28:36,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:28:37,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:28:37,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:28:37,471 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:28:37,471 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:28:37,472 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:28:38,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:28:38,227 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:28:39,228 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:28:40,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:28:45,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:28:50,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:28:55,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:28:56,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:28:56,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:28:57,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:28:58,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:28:58,211 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:28:58,211 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:28:58,212 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:28:58,262 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:28:58,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:28:59,123 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:28:59,263 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:28:59,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:00,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:01,124 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:29:01,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:06,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:29:11,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:29:16,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:29:17,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:29:17,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:18,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:19,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:29:19,131 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:29:19,131 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:29:19,133 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:29:19,301 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:29:19,540 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:20,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:21,303 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:29:21,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:22,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:29:22,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:23,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:24,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:25,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:26,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:27,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:29:27,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:28,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:29,121 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:29:29,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:33,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:29:37,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:29:37,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:38,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:38,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:29:39,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:29:39,083 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:29:39,083 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:29:39,084 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:29:39,320 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:29:39,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:41,322 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:29:44,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:29:49,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:29:50,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:29:50,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:51,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:51,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:29:51,706 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:29:51,706 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:29:51,707 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:29:52,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:29:52,333 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:29:53,334 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:29:54,708 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:29:59,122 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:30:00,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:05,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:06,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:30:06,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:07,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:07,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:30:07,463 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:30:07,463 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:30:07,464 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:30:08,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:08,353 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:30:09,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:09,354 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:30:10,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:15,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:20,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:24,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:30:24,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:25,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:25,635 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:25,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:30:25,679 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:30:25,679 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:30:25,681 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:30:26,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:26,396 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:30:27,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:27,397 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:30:28,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:29,141 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:30:29,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:30,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:31,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:31,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:32,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:33,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:36,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:41,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:46,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:50,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:30:50,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:51,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:51,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:52,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:30:52,215 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:30:52,215 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:30:52,217 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:30:52,417 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:30:52,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:30:53,418 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:30:57,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:30:59,125 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:31:03,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:31:07,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:31:07,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:08,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:08,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:31:08,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:31:08,827 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:31:08,827 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:31:08,828 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:31:09,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:09,448 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:31:10,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:11,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:11,449 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:31:12,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:13,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:14,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:14,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:31:19,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:31:24,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:31:25,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:31:25,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:26,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:27,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:31:27,763 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:31:27,763 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:31:27,765 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:31:27,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:28,464 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:31:28,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:29,127 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:31:29,466 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:31:29,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:30,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:31:30,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:31,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:32,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:33,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:34,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:35,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:31:35,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:36,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:37,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:38,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:39,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:40,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:41,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:31:41,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:42,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:43,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:44,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:45,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:46,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:31:46,263 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:31:46,263 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:31:46,265 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:31:46,529 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:31:46,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:47,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:31:47,530 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:31:47,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:48,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:49,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:50,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:51,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:52,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:31:52,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:53,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:31:57,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:31:59,128 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:32:02,227 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:32:02,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:03,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:03,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:32:03,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:32:03,735 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:32:03,735 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:32:03,737 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:32:04,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:04,544 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:32:04,544 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:32:05,227 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:08,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:32:13,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:32:16,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:32:16,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:17,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:17,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:32:17,731 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:32:17,731 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:32:17,733 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:32:18,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:18,558 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:32:18,558 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:32:19,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:19,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:32:20,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:21,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:22,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:23,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:24,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:25,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:25,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:32:26,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:29,132 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:32:29,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:30,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:31,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:32:31,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:32:31,131 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:32:31,131 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:32:31,133 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:32:31,571 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:32:31,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:32,572 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:32:36,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:32:41,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:32:46,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:32:50,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:32:50,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:51,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:51,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:32:51,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:32:51,380 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:32:51,381 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:32:51,382 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:32:51,604 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:32:52,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:52,605 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:32:53,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:54,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:55,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:56,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:57,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:32:57,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:58,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:59,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:32:59,131 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:33:02,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:33:05,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:33:05,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:06,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:07,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:33:07,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:33:07,359 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:33:07,359 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:33:07,361 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:33:07,634 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:33:07,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:08,635 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:33:08,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:09,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:10,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:11,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:12,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:12,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:33:14,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:15,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:16,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:33:16,571 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:33:16,571 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:33:16,573 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:33:16,643 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:33:16,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:17,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:18,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:33:18,645 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:33:18,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:19,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:20,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:21,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:23,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:33:28,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:33:29,132 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:33:34,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:33:38,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:33:38,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:39,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:39,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:33:39,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:33:39,995 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:33:39,995 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:33:39,997 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:33:40,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:40,675 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:33:40,675 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:33:41,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:42,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:44,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:33:49,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:33:54,377 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:33:54,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:55,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:55,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:33:56,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:33:56,127 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:33:56,127 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:33:56,128 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:33:56,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:56,691 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:33:56,692 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:33:57,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:33:59,136 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:34:01,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:06,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:07,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:34:07,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:08,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:09,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:34:09,239 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:34:09,239 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:34:09,256 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:34:09,731 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:34:09,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:10,732 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:34:10,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:11,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:11,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:12,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:16,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:21,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:26,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:27,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:34:27,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:28,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:29,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:34:29,091 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:34:29,091 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:34:29,094 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:34:29,135 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:34:29,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:29,750 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:34:30,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:30,751 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:34:31,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:32,136 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:32,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:33,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:34,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:35,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:36,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:37,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:37,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:42,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:45,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:34:45,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:46,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:47,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:34:47,207 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:34:47,207 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:34:47,209 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:34:47,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:34:47,765 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:34:48,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:48,766 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:34:53,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:58,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:34:59,136 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:35:03,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:35:03,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:03,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:35:04,949 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:05,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:35:05,287 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:35:05,287 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:35:05,303 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:35:05,794 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:35:05,949 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:06,795 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:35:06,949 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:07,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:08,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:09,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:09,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:35:10,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:11,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:12,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:13,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:14,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:14,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:35:15,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:16,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:17,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:18,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:19,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:20,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:35:20,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:21,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:22,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:35:22,887 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:35:22,887 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:35:22,889 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:35:22,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:23,825 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:35:24,826 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:35:26,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:35:29,139 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:35:32,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:35:33,227 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:35:33,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:34,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:34,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:35:34,555 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:35:34,556 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:35:34,557 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:35:34,837 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:35:34,837 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:35:35,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:36,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:36,839 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:35:37,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:37,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:35:38,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:39,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:40,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:43,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:35:43,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:44,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:44,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:35:44,979 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:35:44,979 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:35:44,981 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:35:45,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:45,847 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:35:46,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:46,848 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:35:47,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:48,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:48,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:35:49,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:50,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:51,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:52,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:53,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:54,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:54,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:35:55,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:56,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:35:59,139 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:35:59,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:00,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:00,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:36:00,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:36:00,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:36:00,695 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:36:00,697 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:36:00,875 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:36:01,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:02,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:02,877 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:36:03,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:04,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:05,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:06,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:06,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:36:11,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:36:11,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:36:11,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:12,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:13,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:36:13,303 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:36:13,303 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:36:13,305 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:36:13,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:13,888 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:36:14,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:14,889 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:36:15,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:16,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:36:21,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:36:26,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:36:29,141 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:36:32,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:36:32,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:36:32,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:33,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:33,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:36:33,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:36:33,747 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:36:33,749 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:36:33,908 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:36:34,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:34,909 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:36:37,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:36:42,171 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:36:42,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:43,171 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:43,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:36:43,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:36:43,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:36:43,623 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:36:43,625 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:36:43,918 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:36:44,171 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:44,919 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:36:45,171 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:46,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:47,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:48,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:36:49,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:36:54,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:36:59,143 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:37:00,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:05,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:10,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:14,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:37:14,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:15,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:15,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:16,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:37:16,107 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:37:16,107 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:37:16,109 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:37:16,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:16,959 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:37:16,959 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:37:17,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:18,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:19,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:20,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:21,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:21,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:22,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:23,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:26,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:28,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:29,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:29,143 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:37:29,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:37:29,560 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:37:29,560 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:37:29,561 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:37:29,972 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:37:30,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:30,973 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:37:32,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:37,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:42,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:45,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:37:45,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:46,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:46,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:37:46,555 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:37:46,555 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:37:46,557 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:37:46,986 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:37:47,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:48,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:37:48,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:48,988 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:37:53,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:58,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:37:59,144 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:38:04,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:38:07,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:38:07,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:08,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:09,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:38:09,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:38:09,283 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:38:09,283 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:38:09,285 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:38:09,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:10,018 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:38:10,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:11,019 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:38:11,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:12,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:13,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:14,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:38:19,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:38:24,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:38:29,145 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:38:30,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:38:32,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:38:32,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:33,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:33,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:38:33,475 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:38:33,475 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:38:33,476 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:38:34,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:34,053 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:38:35,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:35,054 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:38:35,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:38:36,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:37,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:41,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:38:46,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:38:47,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:38:48,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:48,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:49,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:38:49,403 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:38:49,403 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:38:49,404 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:38:49,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:50,070 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:38:50,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:51,071 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:38:51,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:38:51,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:52,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:53,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:54,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:55,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:56,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:56,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:38:57,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:58,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:38:59,149 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:38:59,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:00,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:01,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:02,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:39:02,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:03,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:04,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:05,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:06,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:07,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:39:08,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:39:08,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:08,043 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:39:08,044 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:39:08,045 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:39:08,083 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:39:08,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:09,084 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:39:09,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:10,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:11,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:12,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:39:12,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:13,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:14,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:15,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:16,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:17,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:18,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:39:18,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:19,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:20,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:21,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:39:21,711 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:39:21,711 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:39:21,712 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:39:21,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:22,096 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:39:22,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:23,097 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:39:23,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:39:23,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:24,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:25,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:26,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:27,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:28,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:29,148 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:39:29,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:39:29,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:30,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:31,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:32,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:33,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:39:33,943 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:39:33,943 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:39:33,945 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:39:33,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:34,107 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:39:34,945 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:39:34,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:35,108 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:39:35,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:36,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:39,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:39:42,537 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:39:42,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:43,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:43,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:39:43,995 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:39:43,995 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:39:43,997 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:39:44,116 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:39:44,537 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:39:44,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:39:45,117 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:39:49,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:39:54,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:39:59,149 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:40:00,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:02,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:40:02,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:03,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:04,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:40:04,071 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:40:04,071 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:40:04,072 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:40:04,145 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:40:04,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:05,147 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:40:06,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:11,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:16,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:21,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:22,227 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:40:22,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:23,227 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:23,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:40:23,579 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:40:23,579 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:40:23,581 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:40:24,171 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:40:24,227 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:25,179 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:40:25,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:26,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:26,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:27,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:28,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:29,169 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:40:29,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:30,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:31,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:32,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:32,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:33,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:34,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:35,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:36,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:37,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:37,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:38,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:39,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:39,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:40:39,407 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:40:39,407 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:40:39,409 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:40:40,193 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:40:40,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:41,194 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:40:41,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:42,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:42,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:43,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:44,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:47,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:52,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:54,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:40:54,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:55,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:56,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:40:56,159 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:40:56,159 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:40:56,161 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:40:56,212 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:40:56,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:57,213 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:40:57,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:58,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:40:58,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:40:59,152 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:41:04,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:41:09,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:41:13,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:41:13,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:14,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:14,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:41:15,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:41:15,369 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:41:15,369 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:41:15,371 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:41:15,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:16,231 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:41:16,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:17,232 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:41:17,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:18,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:19,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:20,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:41:24,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:25,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:25,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:41:25,735 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:41:25,735 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:41:25,737 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:41:26,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:26,252 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:41:26,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:41:27,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:27,253 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:41:28,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:29,154 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:41:32,155 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:41:33,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:41:33,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:34,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:35,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:41:35,343 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:41:35,343 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:41:35,345 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:41:35,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:36,262 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:41:36,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:37,263 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:41:37,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:41:37,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:38,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:39,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:40,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:41:42,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:41:47,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:41:52,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:41:57,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:41:59,156 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:41:59,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:41:59,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:00,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:01,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:42:01,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:42:01,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:42:01,253 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:42:01,292 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:42:01,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:02,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:03,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:42:03,294 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:42:03,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:04,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:05,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:06,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:07,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:08,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:42:13,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:42:18,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:42:23,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:42:28,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:42:29,156 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:42:34,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:42:38,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:42:38,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:39,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:39,655 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:42:40,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:42:40,059 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:42:40,059 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:42:40,062 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:42:40,322 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:42:40,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:41,323 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:42:41,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:42,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:43,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:44,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:45,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:45,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:42:46,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:47,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:48,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:49,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:50,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:50,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:42:55,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:42:55,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:42:55,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:56,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:57,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:42:57,327 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:42:57,327 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:42:57,328 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:42:57,349 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:42:57,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:58,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:42:59,158 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:42:59,351 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:43:01,159 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:06,159 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:11,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:16,161 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:21,161 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:26,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:28,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:43:28,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:29,160 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:43:29,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:30,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:43:30,334 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:43:30,334 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:43:30,335 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:43:30,403 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:43:30,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:31,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:31,404 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:43:31,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:32,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:33,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:34,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:35,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:36,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:36,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:37,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:41,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:46,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:49,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:43:49,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:50,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:50,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:43:50,723 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:43:50,723 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:43:50,725 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:43:51,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:51,424 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:43:52,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:43:52,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:53,426 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:43:57,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:43:59,160 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:44:03,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:44:08,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:44:09,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:44:09,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:10,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:11,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:44:11,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:44:11,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:44:11,097 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:44:11,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:11,452 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:44:12,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:13,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:13,455 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:44:14,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:44:14,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:15,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:16,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:17,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:18,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:19,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:44:19,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:20,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:21,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:22,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:22,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:44:22,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:44:22,747 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:44:22,748 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:44:23,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:23,465 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:44:24,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:44:24,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:25,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:25,466 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:44:26,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:27,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:28,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:29,162 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:44:29,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:44:29,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:30,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:34,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:44:36,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:44:36,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:37,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:38,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:44:38,269 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:44:38,269 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:44:38,270 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:44:38,505 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:44:38,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:39,506 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:44:39,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:40,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:44:40,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:41,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:42,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:43,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:44,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:45,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:45,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:44:46,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:47,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:48,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:49,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:50,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:51,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:44:51,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:52,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:44:52,771 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:44:52,771 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:44:52,772 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:44:52,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:53,518 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:44:53,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:54,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:55,521 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:44:55,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:56,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:44:57,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:44:59,163 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:45:03,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:45:05,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:45:05,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:06,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:07,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:45:07,395 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:45:07,395 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:45:07,396 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:45:07,533 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:45:07,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:08,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:45:08,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:09,535 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:45:09,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:10,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:11,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:12,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:13,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:13,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:45:14,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:15,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:16,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:17,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:18,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:19,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:19,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:45:20,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:21,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:22,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:23,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:24,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:25,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:45:25,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:26,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:27,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:28,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:29,173 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:45:29,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:29,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:45:29,847 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:45:29,848 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:45:29,849 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:45:30,566 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:45:30,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:45:30,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:45:31,567 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:45:35,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:45:40,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:45:45,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:45:50,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:45:55,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:45:59,165 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:46:01,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:06,167 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:07,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:46:07,055 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:08,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:08,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:46:08,427 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:46:08,427 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:46:08,430 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:46:08,608 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:46:09,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:09,609 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:46:10,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:11,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:11,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:12,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:13,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:14,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:17,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:17,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:18,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:19,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:46:19,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:46:19,039 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:46:19,041 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:46:19,618 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:46:19,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:20,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:21,620 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:46:22,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:27,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:29,167 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:46:32,168 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:36,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:46:36,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:37,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:37,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:38,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:46:38,215 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:46:38,215 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:46:38,216 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:46:38,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:38,635 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:46:39,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:39,636 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:46:40,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:41,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:42,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:43,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:46:43,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:48,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:53,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:58,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:46:59,172 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:47:04,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:47:04,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:47:04,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:05,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:06,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:47:06,447 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:47:06,447 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:47:06,448 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:47:06,655 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:47:06,655 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:47:06,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:07,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:08,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:09,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:47:09,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:10,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:11,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:12,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:13,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:14,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:47:19,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:47:24,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:47:25,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:47:25,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:26,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:27,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:47:27,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:47:27,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:47:27,097 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:47:27,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:27,687 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:47:28,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:28,688 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:47:29,170 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:47:29,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:30,171 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:47:30,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:31,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:32,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:35,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:47:40,655 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:47:45,655 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:47:49,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:47:49,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:50,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:50,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:47:50,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:47:50,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:47:50,867 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:47:50,868 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:47:51,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:47:51,705 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:47:52,706 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:47:55,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:47:59,171 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:48:01,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:06,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:09,103 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:48:09,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:10,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:10,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:48:10,986 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:48:10,986 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:48:10,987 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:48:11,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:11,722 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:48:11,988 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:12,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:12,723 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:48:13,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:14,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:15,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:16,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:17,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:22,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:23,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:48:23,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:24,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:25,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:48:25,051 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:48:25,051 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:48:25,052 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:48:25,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:25,735 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:48:26,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:26,735 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:48:27,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:28,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:28,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:29,176 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:48:29,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:30,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:31,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:32,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:33,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:33,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:34,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:35,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:36,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:38,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:43,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:48,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:53,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:56,275 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:48:56,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:57,275 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:58,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:48:58,011 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:48:58,011 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:48:58,012 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:48:58,275 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:48:58,760 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:48:58,760 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:48:59,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:48:59,173 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:48:59,275 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:04,175 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:49:09,175 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:49:14,176 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:49:15,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:49:15,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:16,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:16,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:49:16,863 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:49:16,863 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:49:16,866 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:49:17,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:17,775 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:49:18,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:18,777 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:49:19,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:19,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:49:20,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:21,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:22,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:23,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:25,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:49:29,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:49:29,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:29,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:49:30,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:30,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:49:30,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:49:30,531 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:49:30,531 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:49:30,532 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:49:30,788 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:49:30,788 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:49:31,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:32,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:33,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:35,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:49:40,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:49:43,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:49:43,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:44,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:44,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:49:44,551 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:49:44,551 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:49:44,553 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:49:44,801 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:49:44,801 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:49:45,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:45,553 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:49:46,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:47,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:48,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:49,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:49:51,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:49:56,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:49:59,176 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:50:01,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:50:01,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:50:01,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:02,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:02,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:50:02,735 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:50:02,735 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:50:02,736 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:50:02,817 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:50:03,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:04,819 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:50:06,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:50:11,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:50:16,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:50:19,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:50:19,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:20,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:21,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:50:21,435 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:50:21,435 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:50:21,437 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:50:21,835 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:50:21,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:22,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:50:22,836 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:50:22,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:23,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:24,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:27,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:50:29,208 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:50:33,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:50:38,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:50:43,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:50:45,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:50:45,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:46,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:47,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:50:47,187 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:50:47,187 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:50:47,188 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:50:47,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:47,858 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:50:48,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:50:48,859 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:50:49,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:50:54,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:50:59,179 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:51:00,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:01,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:51:01,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:02,332 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:02,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:51:02,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:51:02,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:51:02,813 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:51:02,872 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:51:03,332 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:04,332 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:04,874 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:51:05,332 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:05,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:06,332 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:11,332 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:16,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:21,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:26,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:29,180 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:51:30,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:51:31,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:31,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:32,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:32,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:51:32,461 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:51:32,461 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:51:32,462 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:51:32,896 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:51:32,897 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:51:32,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:33,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:34,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:35,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:36,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:37,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:42,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:47,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:48,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:51:48,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:49,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:50,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:51:50,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:51:50,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:51:50,212 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:51:50,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:50,911 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:51:50,911 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:51:51,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:52,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:53,212 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:53,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:54,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:51:58,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:51:59,182 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:52:00,168 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:52:00,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:01,168 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:01,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:52:01,607 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:52:01,607 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:52:01,608 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:52:01,921 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:52:02,168 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:02,922 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:52:03,168 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:03,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:52:04,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:05,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:06,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:07,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:08,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:09,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:09,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:52:10,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:11,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:12,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:13,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:14,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:15,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:52:15,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:16,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:17,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:18,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:19,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:20,169 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:52:20,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:52:20,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:20,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:52:20,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:52:20,341 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:52:20,940 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:52:20,941 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:52:21,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:25,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:52:29,182 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:52:31,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:52:35,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:52:35,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:36,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:36,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:52:36,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:52:36,491 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:52:36,491 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:52:36,492 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:52:36,972 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:52:36,972 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:52:37,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:38,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:39,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:40,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:41,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:42,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:42,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:52:47,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:52:51,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:52:51,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:52,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:52,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:52:53,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:52:53,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:52:53,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:52:53,198 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:52:53,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:53,985 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:52:54,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:54,986 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:52:55,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:56,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:57,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:58,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:52:58,800 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:52:59,184 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:52:59,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:53:04,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:53:09,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:53:14,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:53:19,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:53:19,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:53:19,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:53:20,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:53:21,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:53:21,219 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:53:21,219 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:53:21,221 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:53:21,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:53:22,013 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:53:22,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:53:23,014 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:53:23,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:53:24,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:53:29,185 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:53:30,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:53:35,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:53:40,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:53:45,188 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:53:45,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:53:45,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:53:46,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:53:47,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:53:47,327 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:53:47,327 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:53:47,328 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:53:47,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:53:48,034 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:53:48,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:53:49,036 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:53:49,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:53:50,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:53:55,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:53:59,186 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:54:01,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:04,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:54:04,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:05,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:05,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:54:05,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:54:05,559 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:54:05,560 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:54:06,050 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:54:06,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:06,561 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:07,051 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:54:07,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:08,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:09,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:10,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:11,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:12,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:17,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:17,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:54:17,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:18,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:19,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:54:19,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:54:19,459 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:54:19,461 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:54:19,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:20,064 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:54:20,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:21,065 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:54:22,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:27,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:29,188 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:54:31,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:54:32,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:32,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:33,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:33,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:54:33,312 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:54:33,312 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:54:33,313 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:54:33,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:34,078 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:54:35,079 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:54:38,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:43,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:48,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:52,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:54:52,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:53,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:53,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:53,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:54:53,583 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:54:53,583 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:54:53,585 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:54:54,098 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:54:54,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:55,099 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:54:55,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:56,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:57,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:58,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:54:59,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:54:59,189 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:55:04,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:55:09,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:55:14,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:55:19,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:55:24,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:55:29,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:55:29,223 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:55:34,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:55:35,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:55:35,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:36,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:37,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:55:37,419 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:55:37,420 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:55:37,421 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:55:37,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:38,173 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:55:38,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:39,174 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:55:39,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:55:39,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:40,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:41,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:42,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:43,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:44,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:44,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:55:45,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:46,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:47,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:55:47,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:55:47,399 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:55:47,400 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:55:47,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:48,182 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:55:48,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:49,184 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:55:49,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:50,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:55:50,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:51,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:55:55,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:55:59,191 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:56:01,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:56:06,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:56:07,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:56:07,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:08,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:09,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:56:09,191 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:56:09,191 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:56:09,193 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:56:09,214 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:56:09,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:11,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:56:11,216 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:56:16,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:56:17,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:56:17,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:18,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:19,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:56:19,183 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:56:19,183 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:56:19,185 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:56:19,223 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:56:19,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:20,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:21,225 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:56:21,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:22,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:56:22,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:23,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:24,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:25,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:26,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:27,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:56:27,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:28,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:29,193 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:56:29,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:30,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:31,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:32,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:33,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:56:33,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:34,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:38,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:56:39,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:40,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:40,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:56:40,487 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:56:40,487 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:56:40,489 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:56:41,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:41,242 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:56:41,242 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:56:42,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:44,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:56:49,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:56:53,295 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:56:53,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:54,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:54,617 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:56:54,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:56:54,659 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:56:54,659 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:56:54,661 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:56:55,273 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:56:55,273 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:56:55,295 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:56,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:57,275 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:56:57,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:58,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:56:59,196 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:56:59,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:00,197 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:00,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:01,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:02,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:03,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:04,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:05,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:05,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:06,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:07,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:08,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:09,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:10,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:57:10,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:10,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:10,307 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:57:10,307 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:57:10,309 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:57:11,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:57:11,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:57:11,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:12,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:13,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:14,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:15,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:16,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:16,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:21,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:21,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:57:21,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:22,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:23,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:57:23,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:57:23,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:57:23,096 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:57:23,302 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:57:23,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:24,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:25,318 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:57:25,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:26,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:26,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:27,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:28,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:29,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:57:29,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:30,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:32,196 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:34,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:35,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:35,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:57:35,681 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:57:35,681 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:57:35,682 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:57:36,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:36,329 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:57:37,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:37,330 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:57:37,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:38,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:42,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:47,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:52,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:55,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:57:55,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:56,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:57,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:57:57,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:57:57,691 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:57:57,691 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:57:57,693 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:57:57,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:57:58,362 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:57:59,196 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:57:59,363 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:58:03,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:58:08,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:58:13,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:58:13,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:58:13,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:14,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:15,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:58:15,515 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:58:15,515 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:58:15,516 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:58:15,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:16,377 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:58:16,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:17,378 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:58:17,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:18,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:58:18,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:19,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:23,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:58:28,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:58:29,228 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:58:33,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:58:33,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:34,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:34,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:58:34,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:58:34,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:58:34,503 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:58:34,504 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:58:35,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:35,398 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:58:35,399 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:58:39,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:58:44,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:58:45,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:58:45,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:46,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:47,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:58:47,239 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:58:47,239 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:58:47,241 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:58:47,417 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:58:47,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:48,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:58:49,419 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:58:50,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:58:55,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:58:59,199 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:59:01,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:06,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:11,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:13,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:59:13,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:14,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:14,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:59:14,723 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:59:14,723 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:59:14,725 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:59:15,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:15,441 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:59:16,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:16,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:17,443 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:59:21,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:26,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:29,200 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 11:59:32,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:32,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:59:32,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:33,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:33,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:59:33,639 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:59:33,639 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:59:33,640 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:59:34,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:34,457 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:59:35,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:35,458 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:59:36,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:37,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:37,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:38,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:42,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:47,212 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:49,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 11:59:49,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:50,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:51,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 11:59:51,355 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 11:59:51,355 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 11:59:51,357 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 11:59:51,472 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 11:59:51,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:52,357 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:52,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:53,474 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 11:59:53,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:54,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:55,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:56,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:57,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 11:59:57,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 11:59:59,202 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:00:03,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:00:05,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:00:05,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:06,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:06,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:00:06,911 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:00:06,911 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:00:06,912 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:00:07,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:07,488 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:00:08,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:08,913 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:00:09,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:09,490 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:00:10,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:11,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:12,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:13,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:14,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:14,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:00:19,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:00:24,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:00:26,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:00:26,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:27,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:27,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:00:27,819 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:00:27,819 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:00:27,821 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:00:28,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:28,505 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:00:29,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:29,205 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:00:29,506 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:00:30,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:30,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:00:31,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:32,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:33,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:34,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:35,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:36,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:00:36,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:37,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:38,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:39,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:40,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:41,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:00:41,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:42,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:43,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:44,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:45,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:46,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:47,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:00:47,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:48,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:49,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:50,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:51,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:52,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:00:52,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:52,540 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:00:52,540 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:00:52,541 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:00:52,542 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:00:53,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:53,525 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:00:53,525 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:00:54,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:55,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:56,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:57,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:00:57,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:58,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:59,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:00:59,204 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:01:00,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:01,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:02,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:03,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:01:03,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:04,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:05,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:06,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:07,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:08,204 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:09,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:01:09,204 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:10,204 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:10,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:01:10,715 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:01:10,715 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:01:10,717 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:01:11,204 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:11,541 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:01:13,543 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:01:14,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:01:19,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:01:24,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:01:29,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:01:29,205 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:01:29,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:01:29,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:30,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:30,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:01:30,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:01:30,695 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:01:30,696 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:01:31,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:31,576 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:01:32,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:01:33,579 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:01:34,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:01:39,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:01:44,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:01:49,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:01:54,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:01:59,223 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:02:00,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:02:05,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:02:07,565 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:02:07,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:08,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:09,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:02:09,243 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:02:09,243 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:02:09,245 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:02:09,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:09,610 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:02:10,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:02:10,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:11,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:11,612 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:02:12,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:13,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:14,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:15,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:15,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:02:16,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:17,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:18,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:19,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:20,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:21,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:02:21,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:22,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:23,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:24,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:25,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:26,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:27,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:02:27,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:28,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:29,208 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:02:29,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:33,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:02:33,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:34,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:35,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:02:35,115 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:02:35,115 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:02:35,117 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:02:35,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:35,636 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:02:36,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:37,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:37,638 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:02:38,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:02:38,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:39,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:40,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:41,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:42,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:43,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:44,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:02:44,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:45,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:45,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:02:45,959 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:02:45,959 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:02:45,960 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:02:46,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:46,661 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:02:47,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:47,662 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:02:48,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:49,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:02:49,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:50,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:51,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:52,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:53,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:54,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:02:54,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:55,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:56,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:57,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:58,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:02:59,209 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:02:59,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:00,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:00,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:01,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:02,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:05,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:10,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:15,212 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:18,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:03:18,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:19,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:19,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:03:19,527 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:03:19,527 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:03:19,529 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:03:19,718 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:03:20,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:20,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:21,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:21,720 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:03:22,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:23,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:24,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:26,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:29,213 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:03:31,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:36,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:37,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:03:37,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:38,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:39,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:03:39,219 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:03:39,219 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:03:39,236 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:03:39,738 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:03:39,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:40,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:41,236 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:41,740 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:03:41,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:42,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:43,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:44,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:45,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:46,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:51,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:55,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:03:55,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:56,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:56,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:03:56,703 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:03:56,703 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:03:56,705 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:03:56,754 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:03:57,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:57,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:03:57,755 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:03:58,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:59,212 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:03:59,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:03:59,757 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:04:00,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:01,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:03,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:08,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:13,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:18,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:23,304 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:23,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:04:23,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:24,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:25,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:04:25,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:04:25,003 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:04:25,005 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:04:25,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:25,795 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:04:26,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:26,796 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:04:27,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:28,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:28,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:29,213 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:04:29,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:30,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:31,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:32,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:33,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:33,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:04:38,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:43,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:48,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:53,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:58,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:04:59,215 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:05:02,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:05:02,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:03,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:03,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:05:03,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:05:03,714 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:05:03,715 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:05:03,716 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:05:03,825 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:05:04,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:04,826 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:05:05,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:06,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:07,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:08,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:09,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:05:14,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:05:19,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:05:24,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:05:29,217 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:05:30,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:05:31,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:05:31,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:32,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:33,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:05:33,107 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:05:33,107 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:05:33,108 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:05:33,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:33,850 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:05:34,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:34,851 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:05:35,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:36,109 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:05:36,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:37,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:38,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:39,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:40,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:41,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:05:41,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:46,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:05:49,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:05:50,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:50,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:51,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:05:51,507 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:05:51,507 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:05:51,509 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:05:51,882 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:05:51,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:52,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:05:52,883 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:05:52,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:05:57,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:05:59,217 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:05:59,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:05:59,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:00,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:01,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:06:01,343 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:06:01,343 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:06:01,345 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:06:01,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:01,893 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:06:02,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:02,894 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:06:03,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:06:03,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:08,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:06:13,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:06:18,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:06:23,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:06:28,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:06:29,218 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:06:30,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:06:30,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:31,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:32,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:06:32,219 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:06:32,219 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:06:32,221 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:06:32,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:32,917 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:06:32,917 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:06:33,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:34,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:06:34,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:35,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:36,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:37,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:38,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:39,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:39,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:06:40,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:41,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:42,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:43,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:44,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:44,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:06:45,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:46,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:47,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:48,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:06:48,715 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:06:48,715 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:06:48,717 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:06:48,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:48,931 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:06:48,931 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:06:49,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:50,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:06:50,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:50,932 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:06:51,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:52,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:06:55,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:06:59,221 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:07:01,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:06,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:07,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:07:07,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:08,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:09,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:07:09,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:07:09,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:07:09,253 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:07:09,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:09,947 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:07:10,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:10,948 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:07:11,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:16,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:21,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:26,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:28,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:07:28,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:29,221 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:07:29,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:30,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:07:30,059 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:07:30,059 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:07:30,060 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:07:30,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:30,967 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:07:30,967 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:07:31,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:32,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:32,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:37,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:42,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:43,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:07:43,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:44,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:44,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:07:44,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:07:44,559 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:07:44,561 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:07:44,980 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:07:44,980 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:07:45,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:47,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:52,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:57,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:07:58,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:07:58,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:59,222 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:07:59,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:07:59,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:07:59,775 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:07:59,775 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:07:59,776 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:07:59,994 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:08:00,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:00,995 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:08:01,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:02,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:08:07,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:08:12,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:08:13,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:08:13,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:14,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:14,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:08:14,495 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:08:14,495 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:08:14,496 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:08:15,008 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:08:15,008 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:08:15,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:18,497 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:08:23,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:08:25,431 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:08:25,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:26,431 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:26,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:08:26,823 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:08:26,823 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:08:26,824 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:08:27,022 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:08:27,432 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:28,432 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:28,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:08:29,024 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:08:29,225 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:08:29,432 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:30,433 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:31,432 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:34,433 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:08:38,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:08:38,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:39,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:39,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:08:40,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:08:40,063 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:08:40,063 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:08:40,064 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:08:40,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:41,035 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:08:41,036 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:08:41,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:42,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:45,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:08:50,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:08:51,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:08:51,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:52,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:52,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:08:52,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:08:52,695 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:08:52,697 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:08:53,048 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:08:53,048 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:08:53,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:54,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:55,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:55,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:08:56,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:57,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:58,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:59,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:08:59,224 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:09:01,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:06,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:06,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:09:06,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:07,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:08,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:09:08,739 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:09:08,739 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:09:08,741 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:09:08,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:09,060 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:09:09,060 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:09:09,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:10,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:11,062 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:09:11,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:11,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:12,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:13,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:14,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:15,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:16,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:16,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:21,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:26,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:29,225 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:09:32,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:34,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:09:34,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:35,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:36,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:09:36,107 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:09:36,107 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:09:36,109 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:09:36,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:37,082 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:09:37,082 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:09:37,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:38,109 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:38,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:39,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:09:43,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:48,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:53,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:58,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:09:59,226 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:10:04,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:10:06,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:10:06,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:10:07,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:10:08,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:10:08,443 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:10:08,443 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:10:08,445 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:10:08,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:10:09,107 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:10:09,107 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:10:09,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:10:14,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:10:19,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:10:23,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:10:23,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:10:24,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:10:24,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:10:24,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:10:24,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:10:24,867 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:10:24,869 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:10:25,121 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:10:25,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:10:26,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:10:27,124 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:10:27,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:10:28,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:10:29,230 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:10:29,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:10:30,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:10:30,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:10:35,231 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:10:40,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:10:45,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:10:50,233 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:10:55,233 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:10:59,229 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:11:00,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:11:00,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:00,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:11:01,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:01,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:11:01,839 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:11:01,839 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:11:01,841 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:11:02,148 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:11:02,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:03,149 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:11:03,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:04,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:05,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:06,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:06,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:11:11,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:12,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:12,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:11:12,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:11:12,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:11:12,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:11:12,813 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:11:13,159 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:11:13,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:14,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:15,162 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:11:15,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:16,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:17,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:18,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:11:21,732 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:22,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:23,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:11:23,184 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:11:23,184 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:11:23,186 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:11:23,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:24,171 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:11:24,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:11:24,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:25,186 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:11:25,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:29,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:11:29,231 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:11:34,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:11:39,233 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:11:44,233 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:11:49,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:11:52,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:11:52,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:53,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:53,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:11:53,899 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:11:53,899 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:11:53,901 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:11:54,207 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:11:54,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:54,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:11:55,222 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:11:55,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:56,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:57,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:11:59,234 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:12:00,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:05,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:10,236 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:11,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:12:11,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:12,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:12,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:12:12,359 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:12:12,359 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:12:12,361 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:12:13,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:13,237 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:12:13,237 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:12:15,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:20,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:25,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:29,233 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:12:31,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:36,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:40,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:12:40,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:41,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:41,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:42,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:12:42,199 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:12:42,199 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:12:42,200 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:12:42,299 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:12:42,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:43,300 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:12:43,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:44,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:45,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:46,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:47,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:47,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:52,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:52,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:12:52,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:53,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:54,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:12:54,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:12:54,503 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:12:54,504 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:12:54,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:55,313 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:12:55,313 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:12:55,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:56,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:57,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:12:57,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:12:59,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:13:03,236 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:13:08,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:13:11,552 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:13:11,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:12,552 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:12,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:13:12,995 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:13:12,995 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:13:12,996 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:13:13,332 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:13:13,552 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:13,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:13:14,553 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:15,334 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:13:18,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:13:23,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:13:28,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:13:29,236 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:13:34,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:13:39,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:13:41,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:13:41,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:42,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:43,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:13:43,215 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:13:43,215 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:13:43,217 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:13:43,359 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:13:43,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:44,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:45,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:13:45,361 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:13:45,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:46,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:47,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:48,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:13:50,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:13:55,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:13:59,237 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:14:01,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:06,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:07,497 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:14:07,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:14:08,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:14:09,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:14:09,102 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:14:09,103 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:14:09,104 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:14:09,380 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:14:09,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:14:10,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:14:11,382 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:14:12,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:17,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:22,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:27,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:28,836 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:14:28,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:14:29,238 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:14:29,836 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:14:30,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:14:30,151 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:14:30,151 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:14:30,153 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:14:30,399 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:14:30,836 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:14:31,400 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:14:31,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:14:32,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:32,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:14:33,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:14:37,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:42,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:47,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:52,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:57,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:14:58,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:14:58,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:14:59,240 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:14:59,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:00,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:15:00,327 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:15:00,327 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:15:00,329 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:15:00,438 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:15:00,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:01,439 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:15:01,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:03,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:15:08,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:15:13,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:15:18,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:15:18,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:15:18,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:19,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:20,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:15:20,394 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:15:20,394 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:15:20,395 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:15:20,456 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:15:20,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:21,457 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:15:21,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:22,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:23,396 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:15:23,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:24,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:28,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:15:29,257 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:15:31,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:15:31,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:32,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:32,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:15:32,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:15:32,635 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:15:32,637 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:15:33,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:33,468 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:15:33,468 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:15:33,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:15:34,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:35,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:36,167 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:37,167 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:38,167 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:39,167 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:15:44,168 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:15:44,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:15:44,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:45,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:46,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:15:46,199 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:15:46,199 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:15:46,201 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:15:46,495 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:15:46,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:47,496 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:15:47,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:48,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:49,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:15:49,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:50,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:51,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:52,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:53,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:54,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:15:54,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:55,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:56,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:57,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:58,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:15:59,242 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:16:00,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:04,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:16:04,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:05,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:05,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:06,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:16:06,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:16:06,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:16:06,341 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:16:06,515 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:16:06,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:07,516 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:16:07,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:08,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:09,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:10,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:11,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:11,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:12,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:13,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:14,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:17,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:19,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:20,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:20,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:16:20,819 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:16:20,819 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:16:20,820 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:16:21,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:21,543 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:16:22,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:22,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:23,545 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:16:27,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:29,243 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:16:33,244 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:38,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:39,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:16:40,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:40,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:41,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:16:41,435 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:16:41,435 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:16:41,437 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:16:41,575 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:16:41,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:16:43,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:43,577 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:16:48,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:53,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:58,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:16:59,246 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:17:03,109 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:17:03,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:04,155 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:17:04,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:04,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:17:04,655 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:17:04,655 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:17:04,656 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:17:05,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:05,595 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:17:05,595 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:17:09,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:17:14,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:17:17,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:17:17,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:18,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:18,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:17:18,999 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:17:18,999 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:17:19,001 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:17:19,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:19,608 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:17:20,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:17:20,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:21,610 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:17:25,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:17:29,246 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:17:30,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:17:35,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:17:37,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:17:37,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:38,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:39,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:17:39,131 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:17:39,131 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:17:39,133 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:17:39,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:39,627 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:17:40,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:41,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:17:41,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:41,628 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:17:42,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:46,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:17:51,336 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:17:56,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:17:58,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:17:58,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:17:59,247 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:17:59,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:00,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:18:00,239 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:18:00,239 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:18:00,241 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:18:00,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:00,647 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:18:01,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:01,648 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:18:02,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:18:02,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:03,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:04,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:05,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:06,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:07,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:18:12,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:18:17,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:18:22,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:18:25,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:18:25,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:26,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:27,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:18:27,077 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:18:27,077 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:18:27,079 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:18:27,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:27,683 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:18:28,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:18:28,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:29,249 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:18:29,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:29,685 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:18:30,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:31,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:33,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:18:36,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:18:36,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:37,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:38,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:18:38,088 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:18:38,088 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:18:38,090 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:18:38,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:38,695 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:18:39,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:18:39,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:39,696 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:18:40,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:41,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:42,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:43,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:44,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:18:44,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:45,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:46,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:47,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:48,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:49,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:18:49,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:50,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:51,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:52,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:54,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:54,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:18:55,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:56,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:18:56,247 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:18:56,247 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:18:56,249 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:18:56,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:56,713 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:18:57,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:57,714 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:18:58,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:18:59,250 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:18:59,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:00,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:00,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:01,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:05,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:06,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:07,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:07,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:19:07,763 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:19:07,763 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:19:07,766 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:19:08,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:08,723 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:19:09,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:09,724 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:19:10,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:10,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:11,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:12,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:13,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:14,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:15,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:16,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:16,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:17,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:18,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:19,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:20,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:21,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:22,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:22,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:23,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:24,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:25,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:26,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:27,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:27,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:28,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:29,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:29,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:19:30,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:31,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:32,251 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:32,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:33,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:34,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:34,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:19:34,915 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:19:34,915 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:19:34,917 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:19:35,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:35,758 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:19:36,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:37,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:37,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:37,760 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:19:38,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:39,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:40,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:41,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:42,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:47,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:52,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:53,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:19:54,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:54,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:55,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:19:55,291 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:19:55,291 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:19:55,292 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:19:55,774 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:19:55,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:56,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:19:57,776 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:19:58,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:19:59,253 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:20:04,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:20:09,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:20:09,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:10,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:10,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:20:10,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:20:10,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:20:10,503 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:20:10,505 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:20:10,790 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:20:11,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:11,791 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:20:12,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:13,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:14,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:15,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:16,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:16,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:20:21,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:20:26,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:20:29,256 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:20:29,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:20:29,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:30,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:30,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:20:30,979 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:20:30,979 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:20:30,980 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:20:31,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:31,809 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:20:31,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:20:32,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:33,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:33,811 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:20:36,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:20:41,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:20:43,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:20:43,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:44,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:44,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:20:44,891 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:20:44,891 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:20:44,893 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:20:45,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:45,822 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:20:46,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:47,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:20:47,824 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:20:47,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:20:52,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:20:57,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:20:59,255 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:21:03,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:21:08,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:21:08,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:21:08,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:09,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:10,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:21:10,419 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:21:10,419 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:21:10,421 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:21:10,842 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:21:10,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:11,842 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:21:11,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:12,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:13,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:21:13,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:14,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:15,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:16,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:18,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:21:23,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:21:28,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:21:29,256 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:21:34,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:21:35,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:21:35,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:36,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:37,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:21:37,328 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:21:37,328 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:21:37,329 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:21:37,862 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:21:37,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:38,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:39,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:21:39,864 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:21:39,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:40,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:41,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:42,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:44,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:21:49,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:21:53,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:21:53,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:54,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:54,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:21:55,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:21:55,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:21:55,039 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:21:55,041 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:21:55,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:55,880 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:21:56,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:57,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:57,882 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:21:58,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:21:59,262 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:21:59,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:00,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:00,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:01,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:02,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:03,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:04,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:05,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:05,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:06,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:07,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:08,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:09,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:10,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:11,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:11,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:12,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:13,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:14,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:15,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:16,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:16,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:17,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:18,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:19,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:19,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:22:19,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:22:19,815 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:22:19,817 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:22:19,899 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:22:20,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:20,900 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:22:21,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:21,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:26,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:29,260 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:22:32,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:37,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:42,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:46,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:22:46,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:47,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:47,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:47,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:22:47,715 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:22:47,715 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:22:47,716 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:22:47,922 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:22:48,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:48,923 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:22:49,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:50,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:51,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:52,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:53,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:53,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:54,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:55,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:56,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:57,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:22:59,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:22:59,261 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:23:00,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:01,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:01,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:23:01,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:23:01,611 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:23:01,613 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:23:01,935 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:23:02,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:02,935 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:23:03,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:04,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:04,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:23:05,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:09,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:23:14,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:23:19,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:23:23,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:23:23,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:24,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:24,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:23:25,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:23:25,091 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:23:25,091 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:23:25,092 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:23:25,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:25,955 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:23:26,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:26,956 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:23:27,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:28,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:29,262 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:23:29,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:30,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:23:30,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:31,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:32,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:33,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:34,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:35,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:35,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:23:40,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:23:44,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:23:44,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:45,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:46,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:23:46,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:23:46,117 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:23:46,117 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:23:46,119 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:23:46,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:46,977 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:23:46,977 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:23:47,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:48,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:23:51,120 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:23:56,120 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:23:59,264 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:24:01,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:24:02,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:24:02,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:03,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:03,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:24:03,735 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:24:03,735 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:24:03,736 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:24:03,991 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:24:04,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:04,992 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:24:05,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:06,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:06,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:24:07,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:12,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:24:17,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:24:22,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:24:23,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:24:23,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:24,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:24,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:24:24,963 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:24:24,963 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:24:24,965 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:24:25,027 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:24:25,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:27,029 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:24:27,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:24:29,265 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:24:33,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:24:38,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:24:40,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:24:40,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:41,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:42,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:24:42,103 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:24:42,103 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:24:42,104 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:24:42,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:43,045 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:24:43,045 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:24:44,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:24:49,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:24:54,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:24:55,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:24:55,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:56,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:56,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:24:56,698 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:24:56,699 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:24:56,700 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:24:57,061 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:24:57,061 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:24:57,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:58,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:24:59,266 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:24:59,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:04,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:09,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:14,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:15,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:25:15,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:25:16,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:25:17,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:25:17,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:25:17,623 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:25:17,625 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:25:17,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:25:18,080 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:25:18,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:25:19,081 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:25:19,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:19,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:25:20,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:25:21,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:25:22,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:25:23,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:25:24,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:29,270 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:25:30,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:35,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:40,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:45,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:50,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:55,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:25:59,269 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:26:00,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:26:00,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:00,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:01,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:01,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:26:01,679 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:26:01,679 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:26:01,681 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:26:02,146 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:26:02,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:03,147 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:26:03,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:04,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:05,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:06,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:11,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:16,244 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:19,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:26:19,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:20,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:21,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:21,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:26:21,571 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:26:21,571 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:26:21,573 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:26:21,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:22,163 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:26:22,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:23,164 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:26:26,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:29,271 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:26:32,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:33,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:26:33,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:34,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:35,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:26:35,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:26:35,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:26:35,197 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:26:35,206 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:26:35,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:36,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:37,208 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:26:37,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:38,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:38,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:39,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:40,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:41,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:42,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:43,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:43,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:44,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:45,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:46,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:47,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:48,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:48,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:49,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:50,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:51,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:52,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:53,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:53,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:54,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:55,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:56,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:26:56,463 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:26:56,463 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:26:56,465 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:26:56,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:57,253 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:26:57,253 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:26:57,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:58,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:26:58,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:26:59,274 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:26:59,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:00,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:01,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:02,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:03,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:04,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:27:09,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:27:14,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:27:15,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:27:15,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:16,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:17,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:27:17,287 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:27:17,287 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:27:17,289 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:27:17,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:18,270 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:27:19,271 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:27:20,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:27:25,291 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:27:26,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:27:26,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:27,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:27,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:27:27,979 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:27:27,979 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:27:27,981 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:27:28,280 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:27:28,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:29,274 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:27:29,281 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:27:29,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:30,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:31,275 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:27:31,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:32,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:33,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:34,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:35,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:36,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:27:36,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:37,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:38,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:41,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:27:43,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:27:43,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:44,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:45,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:27:45,115 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:27:45,115 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:27:45,117 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:27:45,296 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:27:45,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:27:47,117 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:27:47,298 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:27:52,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:27:57,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:27:59,275 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:28:02,276 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:28:06,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:28:06,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:07,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:07,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:28:08,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:28:08,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:28:08,587 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:28:08,589 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:28:08,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:09,330 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:28:09,330 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:28:09,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:13,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:28:18,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:28:23,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:28:28,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:28:29,280 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:28:34,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:28:36,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:28:36,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:37,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:37,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:28:37,813 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:28:37,813 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:28:37,815 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:28:38,357 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:28:38,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:39,358 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:28:39,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:39,815 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:28:40,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:44,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:28:49,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:28:50,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:28:50,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:51,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:51,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:28:51,943 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:28:51,943 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:28:51,944 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:28:52,370 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:28:52,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:53,371 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:28:53,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:54,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:54,945 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:28:55,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:56,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:57,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:28:59,279 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:29:00,280 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:03,140 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:29:03,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:04,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:04,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:29:04,763 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:29:04,763 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:29:04,767 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:29:05,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:05,384 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:29:05,385 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:29:05,767 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:06,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:07,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:08,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:09,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:10,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:11,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:11,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:12,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:13,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:14,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:15,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:16,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:16,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:17,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:17,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:29:17,351 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:29:17,351 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:29:17,352 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:29:17,395 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:29:18,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:19,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:19,398 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:29:20,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:21,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:22,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:22,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:23,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:24,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:25,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:26,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:27,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:27,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:28,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:29,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:29,280 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:29:30,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:31,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:32,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:32,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:33,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:34,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:35,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:36,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:37,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:38,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:38,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:39,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:40,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:41,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:42,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:43,145 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:29:44,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:49,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:54,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:59,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:29:59,283 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:29:59,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:30:00,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:00,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:01,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:30:01,599 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:30:01,599 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:30:01,601 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:30:01,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:02,452 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:30:02,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:03,453 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:30:03,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:04,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:30:04,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:05,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:06,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:09,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:30:11,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:12,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:12,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:30:12,771 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:30:12,771 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:30:12,773 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:30:13,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:13,462 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:30:13,463 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:30:14,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:15,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:15,773 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:30:16,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:17,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:18,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:19,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:20,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:21,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:30:23,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:24,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:25,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:30:25,383 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:30:25,383 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:30:25,384 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:30:25,474 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:30:25,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:26,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:30:26,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:27,476 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:30:27,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:28,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:29,284 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:30:29,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:30,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:32,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:30:37,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:30:38,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:30:38,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:39,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:40,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:30:40,075 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:30:40,075 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:30:40,076 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:30:40,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:40,486 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:30:41,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:30:41,487 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:30:43,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:30:48,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:30:53,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:30:58,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:30:59,284 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:31:03,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:31:08,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:31:13,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:31:15,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:31:15,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:16,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:17,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:31:17,071 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:31:17,071 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:31:17,073 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:31:17,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:17,530 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:31:19,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:31:19,532 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:31:24,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:31:29,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:31:29,285 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:31:32,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:31:32,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:33,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:33,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:31:33,455 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:31:33,455 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:31:33,456 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:31:33,561 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:31:34,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:34,457 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:31:35,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:35,564 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:31:36,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:37,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:38,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:39,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:40,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:40,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:31:41,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:42,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:43,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:44,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:45,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:31:46,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:31:51,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:31:56,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:31:59,289 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:32:01,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:32:01,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:02,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:02,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:32:02,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:32:02,717 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:32:02,717 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:32:02,718 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:32:03,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:03,586 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:32:03,586 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:32:07,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:32:12,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:32:12,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:13,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:13,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:32:13,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:32:13,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:32:13,747 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:32:13,748 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:32:14,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:14,597 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:32:15,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:15,598 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:32:16,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:17,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:19,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:32:24,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:32:27,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:32:27,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:28,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:29,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:32:29,288 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:32:29,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:32:29,327 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:32:29,327 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:32:29,330 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:32:29,610 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:32:29,880 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:30,880 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:31,612 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:32:31,880 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:32,880 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:34,880 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:32:39,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:32:44,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:32:44,932 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:32:44,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:45,932 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:46,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:32:46,491 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:32:46,491 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:32:46,492 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:32:46,625 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:32:46,932 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:47,626 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:32:47,932 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:48,932 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:49,932 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:32:49,932 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:32:54,932 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:32:59,289 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:33:00,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:33:05,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:33:06,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:33:06,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:07,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:07,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:33:07,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:33:07,747 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:33:07,749 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:33:08,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:08,646 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:33:09,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:09,647 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:33:10,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:10,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:33:11,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:12,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:13,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:14,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:15,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:16,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:33:16,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:17,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:21,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:21,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:33:22,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:22,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:33:22,487 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:33:22,487 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:33:22,488 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:33:22,660 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:33:23,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:23,661 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:33:24,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:25,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:26,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:27,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:27,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:33:28,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:29,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:29,307 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:33:30,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:31,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:32,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:33,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:33,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:33:34,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:34,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:33:34,435 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:33:34,435 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:33:34,436 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:33:34,671 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:33:35,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:35,672 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:33:36,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:37,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:38,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:38,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:33:39,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:40,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:41,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:42,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:43,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:44,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:44,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:33:45,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:46,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:47,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:48,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:48,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:33:48,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:33:48,399 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:33:48,400 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:33:48,685 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:33:49,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:49,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:33:49,686 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:33:50,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:51,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:52,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:53,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:54,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:55,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:55,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:33:56,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:57,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:58,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:59,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:33:59,292 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:34:00,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:00,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:01,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:02,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:03,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:04,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:05,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:06,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:06,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:07,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:08,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:09,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:10,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:11,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:11,073 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:11,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:34:11,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:34:11,695 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:34:11,697 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:34:11,719 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:34:12,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:13,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:13,721 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:34:14,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:15,039 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:16,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:21,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:25,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:34:25,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:26,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:26,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:27,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:34:27,023 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:34:27,023 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:34:27,026 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:34:27,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:27,749 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:34:28,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:29,294 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:34:29,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:29,751 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:34:30,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:31,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:32,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:32,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:37,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:42,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:45,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:34:45,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:46,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:47,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:34:47,310 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:34:47,310 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:34:47,312 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:34:47,764 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:34:47,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:48,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:48,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:49,766 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:34:53,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:55,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:34:55,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:56,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:56,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:34:56,847 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:34:56,847 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:34:56,848 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:34:57,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:57,774 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:34:57,774 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:34:58,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:34:58,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:34:59,295 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:34:59,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:00,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:01,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:02,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:04,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:35:05,088 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:06,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:06,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:35:06,499 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:35:06,499 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:35:06,501 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:35:06,783 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:35:07,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:07,784 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:35:08,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:09,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:09,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:35:10,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:14,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:35:19,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:35:24,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:35:24,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:35:24,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:25,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:26,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:35:26,471 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:35:26,471 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:35:26,472 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:35:26,803 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:35:26,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:27,804 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:35:27,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:29,298 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:35:30,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:35:35,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:35:40,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:35:45,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:35:49,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:35:49,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:50,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:50,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:35:50,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:35:50,803 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:35:50,803 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:35:50,805 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:35:50,826 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:35:51,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:51,827 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:35:52,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:35:55,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:35:59,297 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:36:01,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:06,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:09,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:36:09,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:10,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:10,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:36:10,411 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:36:10,411 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:36:10,414 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:36:10,844 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:36:11,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:11,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:11,845 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:36:12,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:13,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:14,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:15,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:17,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:22,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:26,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:36:26,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:27,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:27,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:27,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:36:27,475 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:36:27,475 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:36:27,477 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:36:27,858 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:36:28,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:29,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:29,299 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:36:29,860 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:36:30,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:31,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:32,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:33,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:33,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:34,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:35,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:36,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:37,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:38,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:38,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:38,966 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:36:38,967 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:36:38,967 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:36:38,969 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:36:39,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:39,870 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:36:40,551 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:36:41,872 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:36:44,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:49,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:54,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:59,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:36:59,302 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:37:04,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:37:05,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:37:05,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:37:06,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:37:07,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:37:07,263 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:37:07,263 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:37:07,264 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:37:07,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:37:07,891 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:37:08,833 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:37:09,893 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:37:10,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:37:15,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:37:20,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:37:25,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:37:29,301 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:37:30,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:37:33,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:37:33,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:37:34,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:37:35,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:37:35,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:37:35,351 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:37:35,351 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:37:35,366 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:37:35,914 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:37:35,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:37:36,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:37:37,916 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:37:40,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:37:45,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:37:50,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:37:55,368 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:37:59,302 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:38:01,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:02,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:38:02,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:38:03,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:38:04,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:38:04,446 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:38:04,446 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:38:04,447 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:38:04,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:38:04,938 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:38:05,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:38:05,939 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:38:06,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:06,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:38:07,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:38:08,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:38:11,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:16,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:21,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:26,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:29,306 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:38:32,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:37,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:42,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:46,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:38:46,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:38:47,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:38:47,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:48,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:38:48,387 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:38:48,387 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:38:48,389 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:38:48,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:38:48,989 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:38:49,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:38:49,990 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:38:53,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:58,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:38:59,305 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:39:04,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:39:09,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:39:12,031 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:39:12,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:13,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:13,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:39:13,383 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:39:13,383 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:39:13,385 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:39:14,014 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:39:14,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:14,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:39:15,015 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:39:15,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:16,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:17,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:18,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:20,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:39:25,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:39:28,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:39:28,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:29,307 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:39:29,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:30,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:39:30,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:39:30,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:39:30,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:39:30,197 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:39:30,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:31,029 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:39:31,029 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:39:31,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:32,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:33,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:34,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:35,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:39:40,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:39:45,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:39:47,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:39:47,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:48,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:48,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:39:48,843 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:39:48,843 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:39:48,845 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:39:49,043 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:39:49,044 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:39:49,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:50,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:39:50,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:39:55,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:39:59,308 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:40:01,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:40:03,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:40:03,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:04,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:05,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:40:05,167 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:40:05,167 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:40:05,169 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:40:05,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:06,060 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:40:06,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:07,061 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:40:07,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:40:07,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:08,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:09,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:10,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:11,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:12,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:40:17,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:40:22,772 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:40:27,773 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:40:29,312 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:40:31,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:40:31,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:32,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:32,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:40:33,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:40:33,271 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:40:33,271 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:40:33,272 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:40:33,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:34,082 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:40:34,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:35,083 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:40:35,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:36,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:37,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:38,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:38,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:40:39,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:40,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:41,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:42,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:43,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:44,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:40:46,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:47,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:48,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:40:48,199 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:40:48,199 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:40:48,201 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:40:48,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:49,095 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:40:49,095 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:40:49,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:50,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:40:50,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:51,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:52,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:53,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:54,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:55,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:55,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:40:56,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:57,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:58,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:40:59,311 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:40:59,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:00,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:01,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:01,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:02,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:03,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:04,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:05,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:06,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:06,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:07,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:08,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:41:08,607 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:41:08,607 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:41:08,609 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:41:08,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:09,125 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:41:09,125 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:41:12,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:17,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:22,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:27,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:29,312 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:41:33,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:38,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:43,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:48,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:50,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:41:50,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:51,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:52,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:41:52,523 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:41:52,523 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:41:52,526 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:41:52,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:53,173 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:41:53,173 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:41:53,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:53,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:54,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:55,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:56,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:41:58,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:41:59,315 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:42:04,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:42:09,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:42:14,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:42:16,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:42:16,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:17,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:18,275 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:42:18,276 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:42:18,276 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:42:18,278 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:42:18,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:19,195 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:42:19,195 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:42:19,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:20,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:42:20,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:21,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:22,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:23,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:24,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:25,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:25,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:42:26,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:27,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:28,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:29,315 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:42:29,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:30,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:42:30,643 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:42:30,643 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:42:30,644 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:42:30,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:31,206 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:42:31,207 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:42:31,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:42:31,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:32,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:33,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:34,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:35,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:36,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:42:36,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:40,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:41,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:42,424 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:42:42,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:42:42,471 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:42:42,471 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:42:42,473 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:42:42,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:43,219 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:42:43,219 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:42:43,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:44,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:45,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:46,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:47,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:42:47,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:48,815 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:42:52,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:42:57,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:42:59,316 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:43:02,028 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:43:02,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:03,028 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:03,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:43:03,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:43:03,439 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:43:03,439 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:43:03,441 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:43:04,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:04,254 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:43:05,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:05,255 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:43:06,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:07,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:08,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:09,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:09,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:43:10,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:11,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:12,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:13,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:15,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:43:15,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:16,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:16,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:43:16,519 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:43:16,519 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:43:16,520 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:43:17,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:17,266 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:43:17,267 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:43:18,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:19,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:20,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:43:20,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:21,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:22,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:23,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:25,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:43:28,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:43:28,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:29,317 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:43:29,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:30,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:43:30,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:43:30,280 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:43:30,280 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:43:30,281 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:43:30,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:31,281 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:43:31,281 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:43:31,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:32,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:35,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:43:40,283 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:43:45,283 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:43:49,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:43:50,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:50,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:51,019 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:43:51,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:43:51,771 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:43:51,771 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:43:51,772 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:43:51,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:52,298 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:43:52,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:53,299 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:43:53,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:43:56,773 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:43:59,318 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:44:02,320 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:44:07,103 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:44:07,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:08,103 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:08,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:44:08,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:44:08,551 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:44:08,551 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:44:08,554 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:44:09,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:09,314 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:44:09,315 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:44:10,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:11,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:12,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:13,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:14,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:44:14,104 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:19,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:44:23,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:44:23,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:24,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:24,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:44:25,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:44:25,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:44:25,587 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:44:25,589 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:44:25,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:26,327 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:44:27,328 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:44:29,320 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:44:30,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:44:35,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:44:40,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:44:41,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:44:41,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:42,728 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:43,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:44:43,211 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:44:43,211 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:44:43,213 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:44:43,341 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:44:43,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:44,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:45,358 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:44:45,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:46,213 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:44:46,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:47,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:48,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:49,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:50,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:44:51,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:44:56,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:44:59,321 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:45:02,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:45:07,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:45:11,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:45:11,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:12,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:12,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:45:13,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:45:13,287 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:45:13,287 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:45:13,288 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:45:13,380 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:45:13,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:14,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:15,382 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:45:15,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:16,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:17,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:18,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:18,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:45:19,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:20,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:21,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:22,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:23,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:24,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:45:25,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:26,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:26,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:45:26,907 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:45:26,907 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:45:26,909 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:45:27,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:27,394 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:45:27,394 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:45:28,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:29,324 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:45:29,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:30,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:45:30,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:31,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:32,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:35,864 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:45:40,864 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:45:45,864 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:45:45,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:45:45,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:46,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:47,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:45:47,403 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:45:47,403 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:45:47,404 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:45:47,409 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:45:47,947 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:45:49,411 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:45:51,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:45:56,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:45:59,324 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:46:02,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:46:07,325 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:46:07,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:46:07,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:08,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:09,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:46:09,303 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:46:09,303 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:46:09,319 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:46:09,441 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:46:09,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:10,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:11,443 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:46:11,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:12,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:46:12,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:17,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:46:20,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:46:20,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:21,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:22,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:46:22,179 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:46:22,179 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:46:22,181 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:46:22,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:22,456 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:46:23,181 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:46:23,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:23,457 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:46:24,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:25,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:26,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:27,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:28,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:28,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:46:29,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:29,325 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:46:30,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:34,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:46:39,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:46:44,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:46:49,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:46:54,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:46:59,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:46:59,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:46:59,330 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:46:59,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:00,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:00,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:47:00,555 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:47:00,555 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:47:00,556 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:47:01,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:01,493 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:47:01,493 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:47:02,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:03,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:04,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:05,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:05,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:10,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:10,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:47:10,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:11,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:12,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:47:12,435 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:47:12,435 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:47:12,436 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:47:12,505 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:47:12,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:13,505 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:47:13,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:15,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:20,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:25,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:27,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:47:27,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:28,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:29,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:47:29,147 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:47:29,147 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:47:29,149 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:47:29,328 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:47:29,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:29,519 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:47:30,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:31,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:31,521 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:47:36,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:41,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:46,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:49,800 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:47:49,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:50,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:51,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:47:51,319 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:47:51,319 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:47:51,321 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:47:51,539 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:47:51,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:52,321 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:52,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:53,541 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:47:53,800 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:54,800 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:55,800 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:56,800 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:57,800 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:57,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:47:58,800 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:47:59,329 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:48:02,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:03,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:03,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:48:04,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:48:04,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:48:04,235 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:48:04,236 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:48:04,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:04,554 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:48:05,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:05,554 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:48:06,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:07,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:08,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:09,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:48:14,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:48:19,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:48:22,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:48:22,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:23,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:24,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:48:24,261 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:48:24,261 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:48:24,262 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:48:24,585 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:48:24,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:25,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:48:25,586 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:48:25,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:26,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:29,348 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:48:30,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:48:35,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:48:40,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:48:45,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:48:46,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:48:46,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:47,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:48,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:48:48,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:48:48,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:48:48,097 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:48:48,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:48,607 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:48:49,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:49,608 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:48:50,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:51,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:48:51,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:52,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:53,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:54,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:55,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:56,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:48:56,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:57,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:58,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:59,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:48:59,247 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:48:59,247 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:48:59,263 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:48:59,332 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:48:59,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:48:59,619 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:49:00,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:01,621 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:49:02,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:49:07,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:49:12,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:49:14,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:49:14,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:15,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:16,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:49:16,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:49:16,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:49:16,253 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:49:16,634 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:49:16,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:17,635 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:49:17,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:18,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:49:18,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:19,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:20,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:21,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:23,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:49:28,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:49:29,333 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:49:34,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:49:39,335 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:49:41,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:49:41,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:42,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:42,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:49:42,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:49:42,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:49:42,813 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:49:43,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:43,699 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:49:43,699 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:49:44,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:44,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:49:45,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:46,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:49:50,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:49:55,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:49:59,334 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:49:59,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:49:59,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:00,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:00,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:50:01,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:50:01,139 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:50:01,139 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:50:01,140 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:50:01,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:01,716 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:50:02,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:03,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:03,718 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:50:06,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:50:11,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:50:16,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:50:16,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:17,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:17,043 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:50:17,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:50:17,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:50:17,483 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:50:17,484 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:50:17,732 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:50:18,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:19,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:19,734 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:50:20,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:22,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:50:27,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:50:29,338 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:50:33,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:50:38,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:50:39,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:50:39,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:40,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:40,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:50:40,543 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:50:40,543 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:50:40,545 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:50:40,752 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:50:41,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:41,753 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:50:42,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:43,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:43,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:50:44,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:45,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:46,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:47,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:48,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:49,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:50:49,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:50,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:51,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:52,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:53,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:54,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:55,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:50:55,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:56,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:57,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:58,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:58,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:50:58,875 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:50:58,875 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:50:58,877 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:50:59,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:50:59,337 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:50:59,794 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:50:59,794 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:51:00,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:00,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:01,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:02,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:03,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:04,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:05,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:06,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:06,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:07,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:08,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:11,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:13,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:51:13,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:14,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:15,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:51:15,267 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:51:15,267 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:51:15,269 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:51:15,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:15,823 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:51:16,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:16,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:17,825 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:51:21,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:26,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:29,338 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:51:30,433 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:51:30,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:31,433 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:31,475 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:31,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:51:31,851 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:51:31,851 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:51:31,853 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:51:32,433 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:32,839 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:51:33,840 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:51:36,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:41,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:46,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:49,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:51:49,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:50,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:50,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:51:50,711 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:51:50,711 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:51:50,713 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:51:50,857 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:51:51,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:51:51,858 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:51:52,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:57,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:51:59,342 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:52:01,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:52:01,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:02,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:02,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:52:03,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:52:03,283 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:52:03,283 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:52:03,284 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:52:03,870 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:52:03,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:05,872 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:52:08,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:52:13,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:52:18,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:52:23,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:52:26,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:52:26,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:27,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:28,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:52:28,043 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:52:28,043 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:52:28,045 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:52:28,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:28,937 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:52:29,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:52:29,341 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:52:29,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:29,938 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:52:30,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:34,343 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:52:39,343 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:52:44,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:52:45,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:52:45,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:46,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:47,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:52:47,047 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:52:47,047 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:52:47,050 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:52:47,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:47,969 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:52:48,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:49,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:49,971 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:52:50,051 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:52:50,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:51,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:55,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:52:58,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:52:58,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:59,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:52:59,342 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:52:59,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:52:59,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:52:59,559 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:52:59,561 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:52:59,980 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:53:00,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:01,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:01,561 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:53:01,982 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:53:02,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:03,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:04,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:05,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:06,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:07,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:53:07,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:08,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:12,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:53:17,124 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:53:22,124 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:53:26,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:53:26,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:27,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:27,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:53:27,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:53:27,979 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:53:27,979 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:53:27,981 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:53:28,019 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:53:28,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:29,346 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:53:30,021 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:53:33,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:53:38,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:53:43,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:53:46,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:53:46,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:47,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:47,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:53:47,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:53:47,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:53:47,813 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:53:48,036 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:53:48,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:48,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:53:49,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:50,038 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:53:50,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:51,163 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:52,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:53,163 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:54,163 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:54,163 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:53:55,163 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:56,163 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:57,163 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:58,163 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:59,163 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:53:59,345 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:53:59,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:00,163 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:01,163 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:02,175 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:03,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:03,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:54:03,419 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:54:03,419 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:54:03,420 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:54:04,049 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:54:04,176 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:04,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:06,051 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:54:09,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:14,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:16,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:54:17,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:17,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:18,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:54:18,595 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:54:18,595 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:54:18,597 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:54:18,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:19,077 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:54:19,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:19,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:20,078 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:54:20,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:21,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:22,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:23,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:24,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:25,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:25,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:29,346 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:54:30,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:35,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:39,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:54:39,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:40,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:40,467 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:40,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:54:40,861 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:54:40,861 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:54:40,879 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:54:41,111 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:54:41,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:42,112 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:54:45,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:50,880 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:53,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:54:53,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:54,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:54,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:54:54,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:54:54,559 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:54:54,560 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:54:55,125 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:54:55,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:56,126 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:54:56,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:56,561 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:54:57,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:54:59,348 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:55:02,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:55:04,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:55:05,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:05,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:06,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:55:06,379 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:55:06,379 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:55:06,380 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:55:06,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:07,138 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:55:07,138 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:55:07,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:55:07,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:08,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:09,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:10,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:11,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:12,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:12,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:55:13,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:14,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:18,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:55:23,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:55:24,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:55:24,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:25,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:26,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:55:26,122 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:55:26,122 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:55:26,124 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:55:26,175 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:55:26,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:27,175 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:55:29,124 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:55:29,349 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:55:34,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:55:37,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:55:37,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:38,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:38,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:55:38,675 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:55:38,675 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:55:38,676 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:55:39,187 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:55:39,187 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:55:39,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:39,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:55:40,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:44,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:55:45,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:55:45,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:46,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:47,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:55:47,055 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:55:47,055 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:55:47,057 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:55:47,208 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:55:47,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:48,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:49,210 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:55:49,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:50,057 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:55:50,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:51,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:53,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:54,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:54,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:55:54,775 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:55:54,775 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:55:54,776 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:55:55,230 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:55:55,230 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:55:55,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:55:55,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:56,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:57,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:58,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:59,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:55:59,350 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:56:00,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:00,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:56:05,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:10,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:15,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:20,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:22,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:56:22,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:56:23,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:56:23,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:56:23,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:56:23,635 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:56:23,637 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:56:24,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:56:24,255 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:56:25,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:56:25,270 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:56:25,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:29,352 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:56:31,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:36,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:41,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:46,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:51,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:55,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:56:55,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:56:56,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:56:57,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:56:57,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:56:57,243 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:56:57,243 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:56:57,245 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:56:57,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:56:57,308 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:56:58,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:56:59,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:56:59,310 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:56:59,354 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:57:00,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:01,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:02,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:57:07,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:57:11,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:57:11,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:12,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:12,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:57:13,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:57:13,015 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:57:13,015 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:57:13,017 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:57:13,323 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:57:13,323 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:57:13,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:14,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:15,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:16,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:17,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:18,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:18,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:57:19,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:23,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:57:24,295 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:25,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:25,660 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:57:25,661 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:57:25,661 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:57:25,663 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:57:26,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:26,337 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:57:27,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:27,350 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:57:28,295 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:28,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:57:29,295 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:29,354 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:57:30,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:34,356 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:57:34,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:35,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:36,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:57:36,023 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:57:36,023 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:57:36,025 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:57:36,360 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:57:36,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:37,361 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:57:37,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:38,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:39,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:57:39,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:57:44,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:57:49,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:57:54,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:57:59,356 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:58:00,357 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:05,357 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:10,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:11,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:58:11,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:58:12,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:58:13,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:58:13,295 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:58:13,295 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:58:13,297 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:58:13,401 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:58:13,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:58:14,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:58:15,403 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:58:15,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:58:16,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:16,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:58:21,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:26,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:29,371 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:58:32,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:33,031 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:58:33,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:58:34,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:58:34,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:58:34,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:58:34,503 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:58:34,505 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:58:35,031 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:58:35,439 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:58:35,439 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:58:36,031 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:58:37,031 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:58:37,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:42,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:47,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:52,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:57,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:58:59,358 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:59:03,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:59:08,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:59:10,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:59:10,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:11,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:11,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:59:11,479 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:59:11,479 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:59:11,480 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:59:11,481 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:59:12,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:13,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:13,481 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:59:13,483 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:59:14,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:15,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:19,016 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:59:23,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:59:23,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:24,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:24,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:59:25,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:59:25,455 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:59:25,455 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:59:25,462 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:59:25,493 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:59:25,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:26,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:27,495 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:59:29,360 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 12:59:30,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:59:35,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:59:40,052 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:59:40,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:41,052 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:41,095 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:59:41,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:59:41,715 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:59:41,715 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:59:41,717 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:59:42,052 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:42,507 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:59:43,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:43,508 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:59:46,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:59:51,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:59:52,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 12:59:52,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:53,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:54,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 12:59:54,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 12:59:54,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 12:59:54,252 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 12:59:54,518 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 12:59:54,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:55,519 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 12:59:55,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 12:59:57,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 12:59:59,361 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:00:02,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:07,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:12,363 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:12,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:00:12,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:13,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:14,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:00:14,255 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:00:14,255 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:00:14,256 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:00:14,538 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:00:14,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:15,539 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:00:15,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:16,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:17,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:17,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:18,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:22,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:27,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:29,364 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:00:30,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:00:30,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:31,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:32,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:00:32,215 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:00:32,215 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:00:32,217 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:00:32,556 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:00:32,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:33,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:33,557 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:00:33,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:38,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:43,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:48,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:50,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:00:50,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:51,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:51,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:00:51,909 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:00:51,909 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:00:51,911 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:00:52,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:52,573 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:00:53,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:53,574 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:00:53,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:54,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:00:58,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:00:59,363 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:01:04,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:01:09,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:01:14,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:01:16,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:01:16,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:17,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:17,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:01:17,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:01:17,623 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:01:17,624 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:01:18,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:18,594 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:01:19,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:19,595 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:01:19,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:01:20,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:24,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:01:29,365 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:01:30,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:01:32,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:01:32,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:33,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:34,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:01:34,863 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:01:34,863 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:01:34,864 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:01:34,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:35,607 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:01:35,608 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:01:35,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:01:35,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:36,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:37,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:38,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:39,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:40,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:40,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:01:41,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:42,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:43,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:44,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:45,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:46,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:46,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:01:47,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:48,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:49,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:50,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:51,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:52,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:01:52,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:53,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:54,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:55,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:56,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:57,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:01:57,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:58,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:01:59,368 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:01:59,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:00,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:01,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:02,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:02:02,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:07,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:08,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:08,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:02:09,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:02:09,171 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:02:09,171 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:02:09,173 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:02:09,667 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:02:09,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:10,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:11,669 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:02:11,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:12,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:13,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:13,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:02:14,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:15,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:16,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:17,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:18,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:18,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:02:19,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:20,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:21,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:22,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:23,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:02:28,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:02:29,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:02:34,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:02:36,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:02:36,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:37,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:37,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:02:37,495 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:02:37,495 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:02:37,496 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:02:37,690 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:02:38,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:39,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:39,497 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:02:39,692 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:02:40,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:44,497 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:02:48,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:02:48,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:49,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:49,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:02:50,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:02:50,051 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:02:50,051 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:02:50,052 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:02:50,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:50,703 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:02:51,532 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:02:51,704 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:02:55,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:02:59,369 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:03:00,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:05,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:07,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:03:07,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:08,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:09,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:03:09,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:03:09,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:03:09,369 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:03:09,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:09,733 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:03:10,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:11,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:11,735 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:03:16,371 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:16,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:03:16,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:17,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:18,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:03:18,103 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:03:18,103 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:03:18,105 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:03:18,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:18,742 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:03:19,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:19,743 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:03:22,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:27,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:29,371 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:03:32,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:36,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:03:36,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:37,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:37,495 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:37,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:03:37,915 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:03:37,915 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:03:37,916 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:03:38,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:38,773 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:03:39,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:39,774 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:03:40,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:41,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:42,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:43,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:43,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:44,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:48,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:50,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:03:50,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:51,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:52,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:03:52,163 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:03:52,163 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:03:52,164 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:03:52,786 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:03:52,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:53,787 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:03:53,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:03:54,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:59,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:03:59,371 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:04:04,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:04:09,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:04:14,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:04:19,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:04:20,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:04:20,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:21,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:22,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:04:22,131 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:04:22,131 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:04:22,147 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:04:22,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:22,825 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:04:23,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:23,826 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:04:24,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:25,147 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:04:25,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:26,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:27,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:28,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:29,373 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:04:29,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:30,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:04:30,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:04:30,427 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:04:30,427 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:04:30,429 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:04:30,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:30,833 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:04:31,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:31,834 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:04:32,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:33,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:34,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:35,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:04:35,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:36,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:37,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:38,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:04:40,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:04:45,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:04:50,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:04:55,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:04:59,376 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:05:01,377 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:06,377 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:07,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:05:07,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:08,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:09,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:05:09,411 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:05:09,411 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:05:09,412 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:05:09,905 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:05:09,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:10,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:11,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:11,907 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:05:16,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:16,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:05:16,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:17,447 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:17,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:05:17,871 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:05:17,871 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:05:17,872 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:05:17,913 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:05:18,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:19,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:19,915 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:05:20,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:21,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:21,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:26,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:27,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:27,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:27,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:05:27,887 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:05:27,887 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:05:27,889 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:05:27,923 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:05:28,297 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:29,375 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:05:29,925 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:05:33,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:36,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:05:36,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:37,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:37,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:05:37,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:05:37,611 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:05:37,613 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:05:37,948 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:05:38,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:38,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:39,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:05:39,950 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:05:43,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:48,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:53,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:58,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:05:59,376 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:06:02,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:06:02,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:03,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:03,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:06:03,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:06:03,847 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:06:03,847 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:06:03,848 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:06:03,981 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:06:04,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:05,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:05,982 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:06:08,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:06:13,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:06:18,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:06:22,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:06:22,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:23,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:23,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:06:23,391 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:06:23,391 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:06:23,393 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:06:24,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:24,024 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:06:24,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:06:25,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:26,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:26,026 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:06:29,377 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:06:30,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:06:35,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:06:40,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:06:45,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:06:50,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:06:54,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:06:54,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:55,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:55,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:06:55,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:06:55,975 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:06:55,975 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:06:55,977 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:06:56,050 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:06:56,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:06:58,052 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:06:59,380 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:07:01,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:06,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:10,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:07:10,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:11,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:11,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:12,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:07:12,063 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:07:12,063 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:07:12,079 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:07:12,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:13,063 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:07:13,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:14,064 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:07:14,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:15,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:16,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:17,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:17,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:18,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:22,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:27,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:29,380 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:07:32,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:36,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:07:36,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:37,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:37,447 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:37,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:07:37,448 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:07:37,448 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:07:37,449 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:07:38,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:38,116 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:07:39,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:40,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:40,118 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:07:41,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:07:43,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:48,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:53,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:58,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:07:59,381 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:08:03,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:08:08,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:08:13,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:08:14,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:08:14,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:15,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:16,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:08:16,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:08:16,416 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:08:16,417 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:08:16,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:17,161 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:08:17,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:18,162 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:08:18,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:08:23,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:08:26,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:08:26,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:27,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:28,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:08:28,359 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:08:28,359 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:08:28,360 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:08:28,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:29,182 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:08:29,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:08:29,385 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:08:29,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:30,183 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:08:30,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:34,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:08:39,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:08:41,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:08:41,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:42,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:42,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:08:42,711 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:08:42,711 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:08:42,712 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:08:43,196 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:08:43,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:44,197 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:08:44,236 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:44,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:08:45,236 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:46,236 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:47,236 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:50,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:08:55,236 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:08:56,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:08:56,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:57,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:58,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:08:58,223 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:08:58,223 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:08:58,224 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:08:58,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:08:59,226 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:08:59,226 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:08:59,384 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:08:59,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:00,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:00,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:05,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:10,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:15,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:19,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:09:19,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:20,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:20,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:20,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:09:20,983 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:09:20,983 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:09:20,984 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:09:21,242 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:09:21,243 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:09:21,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:22,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:23,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:24,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:25,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:26,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:26,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:29,385 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:09:31,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:31,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:09:31,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:32,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:33,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:09:33,563 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:09:33,563 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:09:33,565 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:09:33,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:34,255 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:09:34,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:35,256 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:09:35,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:36,565 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:36,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:37,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:41,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:46,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:09:46,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:47,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:47,227 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:47,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:09:47,787 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:09:47,787 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:09:47,789 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:09:48,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:48,268 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:09:49,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:09:49,269 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:09:52,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:57,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:09:59,389 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:10:03,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:10:08,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:10:09,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:10:09,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:10,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:10,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:10:10,631 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:10:10,631 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:10:10,633 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:10:11,180 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:11,287 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:10:11,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:10:13,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:10:18,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:10:23,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:10:28,635 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:10:29,388 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:10:31,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:10:31,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:32,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:33,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:10:33,287 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:10:33,287 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:10:33,288 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:10:33,340 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:10:33,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:34,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:10:35,353 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:10:39,289 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:10:44,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:10:49,291 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:10:50,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:10:50,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:51,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:51,932 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:10:51,932 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:10:51,933 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:10:51,934 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:10:52,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:52,379 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:10:53,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:53,380 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:10:54,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:54,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:10:55,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:56,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:57,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:58,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:59,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:10:59,390 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:11:00,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:00,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:01,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:03,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:04,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:05,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:11:05,159 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:11:05,159 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:11:05,162 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:11:05,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:05,393 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:11:05,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:06,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:07,395 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:11:10,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:15,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:20,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:24,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:11:24,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:25,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:25,435 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:26,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:11:26,155 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:11:26,155 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:11:26,157 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:11:26,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:26,413 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:11:27,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:27,414 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:11:28,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:29,391 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:11:29,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:30,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:31,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:31,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:32,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:36,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:36,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:37,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:38,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:11:38,247 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:11:38,247 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:11:38,248 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:11:38,441 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:11:38,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:39,441 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:11:42,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:47,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:52,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:54,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:11:54,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:55,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:56,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:11:56,523 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:11:56,523 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:11:56,525 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:11:56,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:57,457 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:11:57,457 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:11:57,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:11:57,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:58,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:11:59,394 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:11:59,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:00,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:01,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:02,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:12:07,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:12:07,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:12:08,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:08,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:09,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:12:09,595 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:12:09,595 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:12:09,597 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:12:09,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:10,483 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:12:11,484 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:12:13,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:12:18,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:12:22,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:12:22,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:23,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:23,767 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:12:24,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:12:24,079 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:12:24,079 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:12:24,081 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:12:24,495 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:12:24,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:25,537 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:12:29,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:12:29,394 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:12:34,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:12:35,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:12:35,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:36,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:36,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:12:36,923 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:12:36,923 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:12:36,924 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:12:37,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:37,549 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:12:37,550 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:12:39,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:12:44,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:12:49,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:12:54,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:12:56,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:12:57,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:57,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:58,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:12:58,691 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:12:58,691 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:12:58,693 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:12:58,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:12:59,394 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:12:59,569 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:12:59,569 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:12:59,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:00,396 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:00,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:01,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:02,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:03,997 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:05,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:10,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:12,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:13:12,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:13,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:14,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:13:14,363 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:13:14,363 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:13:14,365 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:13:14,583 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:13:14,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:15,584 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:13:15,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:16,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:16,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:17,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:21,875 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:25,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:13:25,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:26,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:26,604 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:13:26,605 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:13:26,605 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:13:26,606 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:13:26,607 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:13:27,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:27,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:27,608 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:13:28,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:29,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:29,411 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:13:30,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:31,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:32,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:33,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:33,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:34,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:38,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:39,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:13:39,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:40,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:41,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:13:41,427 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:13:41,427 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:13:41,429 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:13:41,622 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:13:41,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:43,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:43,624 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:13:48,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:53,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:56,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:13:56,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:57,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:58,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:13:58,223 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:13:58,223 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:13:58,224 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:13:58,637 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:13:58,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:13:59,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:13:59,397 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:13:59,638 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:13:59,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:00,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:01,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:02,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:03,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:04,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:04,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:14:07,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:08,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:08,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:14:08,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:14:08,747 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:14:08,749 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:14:09,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:09,654 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:14:09,654 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:14:10,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:10,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:14:11,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:12,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:13,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:14,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:15,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:16,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:16,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:14:17,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:18,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:19,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:20,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:21,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:21,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:14:22,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:23,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:24,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:25,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:26,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:27,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:14:27,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:27,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:14:27,827 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:14:27,827 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:14:27,830 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:14:28,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:28,685 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:14:29,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:14:29,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:29,686 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:14:33,400 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:14:38,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:14:39,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:14:39,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:40,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:40,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:14:40,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:14:40,623 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:14:40,626 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:14:40,696 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:14:41,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:41,697 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:14:42,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:43,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:14:48,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:14:51,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:14:51,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:52,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:53,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:14:53,271 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:14:53,271 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:14:53,273 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:14:53,709 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:14:53,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:54,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:14:54,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:55,725 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:14:55,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:56,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:57,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:58,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:14:59,402 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:14:59,403 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:14:59,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:00,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:01,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:02,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:03,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:04,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:15:09,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:15:10,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:15:10,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:11,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:12,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:15:12,211 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:15:12,211 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:15:12,213 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:15:12,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:12,742 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:15:13,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:13,742 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:15:14,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:15,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:15:15,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:16,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:17,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:20,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:15:20,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:21,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:22,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:15:22,291 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:15:22,291 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:15:22,293 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:15:22,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:22,752 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:15:23,752 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:15:26,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:15:29,401 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:15:31,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:15:34,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:15:34,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:35,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:35,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:15:35,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:15:35,559 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:15:35,561 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:15:35,764 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:15:36,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:36,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:15:37,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:15:37,766 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:15:41,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:15:46,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:15:51,563 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:15:56,564 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:15:59,403 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:16:02,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:16:05,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:16:05,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:06,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:06,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:16:06,944 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:16:06,944 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:16:06,945 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:16:07,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:07,807 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:16:07,807 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:16:07,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:16:08,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:09,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:10,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:11,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:13,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:16:18,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:16:23,520 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:16:28,520 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:16:29,406 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:16:34,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:16:35,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:16:35,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:36,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:36,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:16:36,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:16:36,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:16:36,720 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:16:36,835 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:16:37,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:37,836 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:16:38,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:39,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:39,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:16:40,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:41,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:42,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:43,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:44,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:45,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:16:45,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:46,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:47,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:49,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:50,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:50,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:16:50,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:16:50,679 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:16:50,679 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:16:50,680 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:16:50,848 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:16:51,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:51,849 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:16:52,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:53,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:54,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:55,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:56,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:16:56,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:57,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:58,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:59,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:16:59,405 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:17:00,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:01,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:02,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:17:02,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:03,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:04,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:05,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:07,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:17:11,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:17:11,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:12,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:12,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:17:13,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:17:13,143 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:17:13,143 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:17:13,145 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:17:13,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:13,896 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:17:13,896 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:17:14,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:15,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:16,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:17,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:18,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:17:18,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:19,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:23,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:17:28,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:17:29,406 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:17:34,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:17:39,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:17:39,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:17:39,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:40,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:41,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:17:41,183 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:17:41,183 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:17:41,184 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:17:41,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:17:41,920 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:17:41,920 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:17:43,922 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:17:45,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:17:50,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:17:55,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:17:59,408 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:18:00,409 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:05,409 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:10,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:13,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:18:14,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:14,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:15,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:18:15,371 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:18:15,371 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:18:15,402 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:18:15,968 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:18:15,996 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:16,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:17,970 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:18:21,403 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:26,403 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:29,410 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:18:31,411 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:36,412 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:41,412 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:43,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:18:43,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:44,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:45,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:18:45,331 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:18:45,331 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:18:45,347 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:18:45,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:45,994 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:18:46,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:47,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:47,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:47,996 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:18:48,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:49,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:50,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:51,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:52,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:52,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:53,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:54,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:18:58,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:18:59,410 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:19:04,411 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:19:08,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:19:08,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:09,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:09,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:19:10,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:19:10,035 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:19:10,035 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:19:10,036 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:19:10,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:11,018 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:19:11,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:12,020 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:19:12,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:13,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:15,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:19:20,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:19:25,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:19:26,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:19:26,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:27,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:28,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:19:28,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:19:28,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:19:28,340 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:19:28,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:29,037 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:19:29,411 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:19:29,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:30,038 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:19:30,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:31,412 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:19:36,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:19:41,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:19:46,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:19:46,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:47,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:47,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:19:47,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:19:47,607 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:19:47,607 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:19:47,608 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:19:48,057 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:19:48,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:49,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:50,058 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:19:50,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:51,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:52,204 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:53,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:53,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:19:54,204 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:55,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:56,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:57,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:58,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:19:58,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:19:59,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:20:03,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:20:08,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:20:13,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:20:14,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:20:14,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:15,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:15,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:20:15,793 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:20:15,794 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:20:15,795 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:20:16,078 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:20:16,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:17,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:18,080 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:20:18,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:18,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:20:23,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:20:28,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:20:29,414 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:20:33,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:20:33,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:33,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:20:34,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:35,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:20:35,263 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:20:35,263 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:20:35,266 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:20:35,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:36,096 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:20:36,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:37,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:38,098 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:20:39,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:20:44,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:20:49,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:20:49,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:20:49,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:50,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:51,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:20:51,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:20:51,399 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:20:51,401 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:20:51,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:52,109 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:20:52,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:53,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:54,111 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:20:54,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:20:54,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:55,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:56,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:20:59,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:20:59,416 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:04,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:09,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:09,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:21:09,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:10,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:11,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:21:11,251 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:21:11,251 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:21:11,253 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:21:11,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:12,126 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:21:12,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:13,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:14,127 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:21:14,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:14,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:15,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:16,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:17,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:18,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:19,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:20,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:25,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:29,416 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:21:31,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:33,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:21:33,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:34,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:34,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:21:34,419 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:21:34,419 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:21:34,420 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:21:35,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:35,195 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:21:36,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:36,196 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:21:36,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:37,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:38,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:39,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:42,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:47,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:52,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:52,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:21:52,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:53,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:53,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:21:53,708 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:21:53,708 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:21:53,710 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:21:54,213 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:21:54,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:21:56,214 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:21:57,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:21:59,418 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:22:03,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:22:03,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:22:04,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:04,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:05,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:22:05,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:22:05,459 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:22:05,461 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:22:05,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:06,224 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:22:06,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:07,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:08,226 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:22:08,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:22:08,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:09,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:10,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:11,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:12,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:13,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:22:13,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:14,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:15,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:16,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:17,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:18,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:22:18,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:21,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:22,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:22,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:22:22,843 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:22:22,843 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:22:22,845 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:22:23,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:23,258 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:22:23,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:22:24,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:24,258 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:22:25,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:26,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:27,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:28,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:28,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:22:29,419 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:22:34,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:22:39,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:22:44,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:22:44,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:22:44,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:45,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:45,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:22:45,875 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:22:45,875 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:22:45,877 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:22:46,307 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:22:46,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:48,310 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:22:49,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:22:54,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:22:56,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:22:56,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:57,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:57,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:22:57,975 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:22:57,975 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:22:57,977 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:22:58,318 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:22:58,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:22:59,319 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:22:59,421 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:22:59,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:00,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:00,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:01,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:02,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:03,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:04,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:05,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:05,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:06,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:07,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:08,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:09,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:10,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:11,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:11,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:12,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:13,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:14,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:14,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:23:14,690 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:23:14,690 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:23:14,692 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:23:15,347 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:23:15,347 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:23:15,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:16,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:16,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:17,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:18,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:21,595 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:26,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:23:26,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:27,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:27,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:27,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:23:27,947 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:23:27,947 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:23:27,949 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:23:28,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:28,360 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:23:29,361 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:23:29,423 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:23:32,425 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:37,425 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:42,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:47,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:51,051 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:23:51,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:52,051 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:52,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:52,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:23:52,755 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:23:52,755 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:23:52,757 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:23:53,051 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:53,379 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:23:53,380 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:23:54,052 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:55,052 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:56,052 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:57,052 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:23:58,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:23:59,423 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:24:03,425 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:24:08,425 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:24:13,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:24:17,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:24:17,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:24:18,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:24:18,564 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:24:18,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:24:18,607 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:24:18,607 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:24:18,608 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:24:19,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:24:19,400 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:24:19,400 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:24:23,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:24:28,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:24:29,424 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:24:34,425 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:24:35,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:24:35,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:24:36,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:24:36,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:24:36,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:24:36,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:24:36,813 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:24:37,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:24:37,415 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:24:37,415 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:24:38,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:24:39,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:24:44,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:24:49,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:24:52,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:24:52,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:24:53,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:24:54,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:24:54,539 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:24:54,539 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:24:54,541 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:24:54,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:24:55,447 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:24:55,447 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:24:55,541 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:24:59,429 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:25:01,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:06,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:06,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:25:06,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:07,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:08,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:25:08,083 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:25:08,083 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:25:08,085 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:25:08,473 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:25:08,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:09,474 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:25:09,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:10,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:11,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:11,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:16,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:21,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:26,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:29,427 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:25:29,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:25:29,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:30,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:31,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:25:31,199 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:25:31,199 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:25:31,201 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:25:31,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:31,505 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:25:31,506 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:25:32,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:32,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:37,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:42,203 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:46,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:25:46,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:47,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:47,835 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:48,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:25:48,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:48,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:25:48,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:25:48,813 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:25:49,521 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:25:49,521 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:25:49,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:50,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:25:53,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:58,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:25:59,428 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:26:00,136 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:26:00,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:01,136 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:01,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:26:01,671 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:26:01,671 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:26:01,673 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:26:02,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:02,531 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:26:03,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:03,532 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:26:04,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:04,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:26:05,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:06,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:07,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:10,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:26:15,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:26:20,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:26:22,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:26:22,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:23,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:24,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:26:24,353 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:26:24,353 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:26:24,354 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:26:24,566 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:26:24,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:25,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:26:25,567 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:26:25,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:26,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:29,431 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:26:30,432 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:26:35,433 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:26:40,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:26:45,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:26:45,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:26:45,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:46,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:47,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:26:47,303 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:26:47,303 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:26:47,305 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:26:47,596 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:26:47,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:48,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:49,598 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:26:49,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:50,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:26:50,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:51,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:52,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:53,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:54,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:55,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:56,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:26:56,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:26:59,431 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:27:02,432 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:04,835 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:27:04,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:05,835 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:06,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:27:06,499 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:27:06,499 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:27:06,500 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:27:06,643 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:27:06,836 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:07,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:07,644 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:27:07,836 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:12,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:17,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:22,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:23,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:27:23,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:24,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:25,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:27:25,035 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:27:25,035 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:27:25,036 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:27:25,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:25,658 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:27:25,658 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:27:26,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:27,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:28,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:29,432 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:27:33,433 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:38,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:42,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:27:42,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:43,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:43,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:44,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:27:44,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:27:44,415 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:27:44,417 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:27:44,675 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:27:44,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:45,677 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:27:45,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:46,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:27:49,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:54,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:59,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:27:59,434 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:28:01,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:28:01,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:02,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:02,523 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:28:02,524 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:28:02,524 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:28:02,525 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:28:02,704 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:28:03,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:03,705 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:28:04,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:28:09,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:28:14,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:28:19,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:28:24,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:28:29,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:28:29,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:29,453 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:28:30,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:30,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:28:30,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:28:30,479 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:28:30,479 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:28:30,480 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:28:30,754 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:28:31,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:31,755 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:28:32,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:33,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:34,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:35,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:36,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:28:36,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:41,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:28:46,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:28:46,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:28:46,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:47,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:48,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:28:48,607 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:28:48,607 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:28:48,609 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:28:48,784 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:28:48,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:49,785 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:28:49,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:50,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:51,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:28:51,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:52,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:53,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:54,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:55,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:56,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:28:56,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:28:59,454 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:29:02,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:29:06,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:29:06,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:07,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:07,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:29:07,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:29:07,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:29:07,635 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:29:07,636 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:29:07,820 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:29:08,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:09,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:09,823 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:29:10,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:11,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:12,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:13,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:29:13,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:14,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:15,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:16,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:17,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:18,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:19,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:29:19,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:20,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:21,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:22,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:23,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:24,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:25,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:29:25,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:26,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:27,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:28,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:29,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:29,455 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:29:30,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:30,456 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:29:31,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:32,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:33,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:34,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:35,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:36,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:29:36,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:37,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:41,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:29:45,140 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:29:45,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:46,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:46,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:29:46,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:29:46,915 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:29:46,915 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:29:46,917 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:29:47,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:47,876 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:29:47,877 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:29:48,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:29:51,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:29:56,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:29:59,473 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:30:02,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:30:05,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:30:06,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:06,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:07,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:30:07,370 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:30:07,371 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:30:07,372 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:30:07,921 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:30:07,921 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:30:07,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:08,372 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:30:08,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:09,923 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:30:09,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:10,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:11,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:12,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:13,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:13,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:30:14,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:15,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:16,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:17,975 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:18,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:19,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:19,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:30:20,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:21,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:22,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:24,976 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:30:29,457 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:30:30,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:30:35,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:30:40,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:30:44,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:30:44,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:45,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:45,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:30:46,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:30:46,015 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:30:46,015 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:30:46,018 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:30:46,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:46,952 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:30:47,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:47,953 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:30:48,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:49,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:50,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:51,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:30:51,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:30:56,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:30:59,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:31:02,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:31:03,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:31:03,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:04,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:05,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:31:05,335 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:31:05,335 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:31:05,336 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:31:05,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:05,968 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:31:06,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:07,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:07,970 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:31:08,337 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:31:08,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:09,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:10,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:11,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:12,891 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:13,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:31:18,892 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:31:23,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:31:23,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:24,179 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:24,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:31:24,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:31:24,615 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:31:24,615 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:31:24,617 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:31:24,997 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:31:25,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:26,030 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:31:29,477 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:31:29,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:31:34,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:31:39,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:31:39,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:31:39,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:40,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:41,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:31:41,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:31:41,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:31:41,097 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:31:41,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:42,046 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:31:42,047 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:31:42,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:43,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:44,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:44,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:31:45,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:46,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:47,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:48,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:31:49,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:31:54,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:31:59,461 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:32:00,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:03,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:32:03,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:04,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:04,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:32:04,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:32:04,867 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:32:04,869 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:32:05,065 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:32:05,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:05,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:06,066 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:32:06,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:07,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:08,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:09,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:10,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:11,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:11,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:12,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:13,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:14,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:15,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:16,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:17,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:22,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:27,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:29,464 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:32:32,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:37,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:32:37,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:38,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:38,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:38,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:32:38,619 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:32:38,619 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:32:38,620 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:32:39,105 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:32:39,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:40,106 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:32:40,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:43,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:48,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:53,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:55,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:32:55,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:56,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:56,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:32:56,999 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:32:56,999 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:32:57,001 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:32:57,125 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:32:57,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:58,126 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:32:58,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:32:59,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:32:59,464 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:32:59,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:00,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:04,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:33:09,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:33:14,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:33:15,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:33:15,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:16,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:17,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:33:17,419 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:33:17,419 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:33:17,421 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:33:17,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:18,178 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:33:18,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:19,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:20,180 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:33:20,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:33:20,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:21,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:22,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:25,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:33:29,468 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:33:31,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:33:35,661 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:33:35,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:36,661 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:36,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:33:37,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:33:37,047 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:33:37,047 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:33:37,048 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:33:37,197 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:33:37,661 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:38,198 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:33:42,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:33:47,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:33:48,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:33:48,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:49,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:49,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:33:49,884 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:33:49,885 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:33:49,886 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:33:50,210 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:33:50,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:51,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:52,212 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:33:52,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:52,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:33:53,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:54,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:33:58,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:33:59,466 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:34:03,468 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:34:08,468 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:34:13,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:34:14,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:34:14,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:15,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:16,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:34:16,119 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:34:16,119 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:34:16,122 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:34:16,248 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:34:16,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:17,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:18,250 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:34:18,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:19,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:34:19,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:24,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:34:29,468 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:34:29,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:34:29,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:30,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:30,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:34:31,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:34:31,151 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:34:31,151 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:34:31,153 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:34:31,263 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:34:31,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:32,264 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:34:32,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:33,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:34,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:35,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:36,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:34:41,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:34:46,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:34:50,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:34:50,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:51,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:51,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:34:51,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:34:51,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:34:51,695 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:34:51,696 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:34:52,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:52,283 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:34:53,259 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:54,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:54,285 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:34:55,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:56,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:57,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:57,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:34:58,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:59,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:34:59,471 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:35:00,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:01,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:02,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:03,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:03,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:35:04,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:05,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:06,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:07,260 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:08,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:09,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:09,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:35:10,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:11,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:11,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:35:11,639 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:35:11,639 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:35:11,641 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:35:12,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:12,316 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:35:13,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:14,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:14,318 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:35:14,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:35:15,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:16,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:17,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:18,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:19,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:20,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:35:20,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:21,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:22,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:23,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:24,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:25,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:26,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:35:26,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:27,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:28,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:29,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:29,470 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:35:30,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:31,261 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:35:31,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:32,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:33,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:34,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:35,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:36,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:35:36,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:37,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:38,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:39,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:40,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:41,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:42,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:35:42,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:43,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:44,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:45,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:46,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:47,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:35:47,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:48,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:48,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:35:48,527 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:35:48,527 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:35:48,529 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:35:49,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:49,350 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:35:50,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:50,351 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:35:51,264 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:52,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:35:56,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:35:56,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:57,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:57,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:35:58,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:35:58,087 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:35:58,087 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:35:58,089 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:35:58,360 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:35:58,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:35:59,472 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:35:59,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:36:00,362 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:36:00,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:36:03,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:36:08,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:36:13,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:36:16,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:36:16,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:36:17,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:36:17,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:36:17,807 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:36:17,807 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:36:17,808 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:36:18,378 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:36:18,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:36:18,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:36:20,380 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:36:23,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:36:28,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:36:29,476 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:36:34,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:36:36,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:36:36,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:36:37,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:36:37,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:36:37,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:36:37,611 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:36:37,613 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:36:38,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:36:38,397 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:36:39,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:36:39,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:36:40,399 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:36:44,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:36:49,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:36:54,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:36:59,479 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:37:00,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:03,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:37:03,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:04,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:05,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:37:05,239 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:37:05,239 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:37:05,241 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:37:05,444 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:37:05,444 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:37:05,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:06,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:11,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:14,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:37:14,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:15,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:15,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:37:15,859 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:37:15,860 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:37:15,861 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:37:16,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:16,455 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:37:16,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:17,308 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:17,456 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:37:21,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:26,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:29,479 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:37:32,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:36,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:37:36,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:37,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:37,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:38,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:37:38,087 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:37:38,087 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:37:38,089 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:37:38,475 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:37:38,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:39,476 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:37:39,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:40,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:41,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:37:43,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:48,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:53,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:58,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:37:59,480 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:38:04,481 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:38:09,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:38:09,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:10,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:10,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:38:10,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:38:10,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:38:10,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:38:10,928 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:38:11,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:11,514 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:38:11,515 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:38:12,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:13,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:14,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:16,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:38:21,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:38:26,269 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:38:27,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:38:27,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:28,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:29,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:38:29,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:38:29,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:38:29,369 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:38:29,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:38:29,547 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:38:29,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:30,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:31,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:38:31,549 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:38:31,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:32,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:33,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:34,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:35,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:36,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:36,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:38:41,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:38:46,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:38:51,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:38:55,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:38:55,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:56,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:56,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:38:57,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:38:57,523 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:38:57,523 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:38:57,525 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:38:57,617 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:38:57,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:58,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:38:59,482 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:38:59,619 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:38:59,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:00,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:01,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:02,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:02,939 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:39:07,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:39:12,940 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:39:17,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:39:22,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:39:23,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:39:24,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:24,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:25,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:39:25,651 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:39:25,651 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:39:25,658 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:39:25,658 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:39:25,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:26,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:27,661 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:39:27,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:28,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:39:28,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:29,484 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:39:34,485 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:39:36,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:39:36,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:37,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:37,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:39:37,875 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:39:37,875 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:39:37,877 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:39:38,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:38,672 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:39:39,673 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:39:39,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:39:44,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:39:49,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:39:54,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:39:58,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:39:58,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:39:59,487 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:39:59,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:00,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:00,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:40:00,347 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:40:00,347 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:40:00,349 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:40:00,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:00,691 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:40:01,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:01,692 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:40:02,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:05,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:10,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:13,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:40:13,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:14,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:15,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:40:15,051 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:40:15,051 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:40:15,052 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:40:15,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:15,706 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:40:15,707 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:40:16,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:16,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:17,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:18,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:19,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:20,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:21,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:21,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:26,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:29,486 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:40:31,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:35,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:40:35,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:36,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:36,492 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:36,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:40:36,539 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:40:36,539 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:40:36,541 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:40:36,726 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:40:37,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:37,727 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:40:38,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:39,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:40,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:41,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:42,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:42,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:43,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:44,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:45,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:46,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:40:48,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:53,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:58,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:40:59,488 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:41:03,489 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:41:07,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:41:07,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:08,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:08,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:41:08,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:41:08,931 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:41:08,932 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:41:08,935 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:41:09,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:09,766 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:41:09,766 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:41:10,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:13,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:41:18,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:41:23,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:41:23,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:24,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:24,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:41:25,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:41:25,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:41:25,459 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:41:25,460 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:41:25,796 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:41:25,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:26,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:27,798 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:41:27,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:28,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:29,489 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:41:30,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:41:35,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:41:40,491 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:41:43,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:41:43,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:44,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:45,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:41:45,303 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:41:45,303 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:41:45,305 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:41:45,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:45,814 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:41:45,814 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:41:46,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:41:46,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:47,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:48,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:49,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:50,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:51,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:51,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:41:52,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:53,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:54,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:55,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:56,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:57,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:41:57,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:58,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:59,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:41:59,492 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:42:00,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:01,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:02,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:02,493 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:42:03,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:04,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:05,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:06,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:07,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:08,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:42:08,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:09,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:10,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:11,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:11,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:42:11,511 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:42:11,511 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:42:11,513 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:42:11,851 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:42:12,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:13,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:42:13,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:13,853 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:42:18,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:42:23,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:42:28,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:42:29,491 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:42:33,493 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:42:38,493 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:42:40,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:42:40,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:41,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:41,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:42:41,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:42:41,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:42:41,797 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:42:41,876 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:42:42,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:43,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:43,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:42:43,878 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:42:44,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:45,409 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:46,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:47,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:48,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:49,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:49,409 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:42:50,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:51,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:55,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:55,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:42:56,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:56,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:42:56,655 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:42:56,655 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:42:56,657 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:42:56,890 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:42:57,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:57,891 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:42:58,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:59,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:42:59,493 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:43:00,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:00,409 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:01,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:05,409 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:10,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:12,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:43:12,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:13,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:14,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:43:14,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:43:14,399 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:43:14,401 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:43:14,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:14,908 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:43:15,909 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:43:16,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:21,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:26,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:29,499 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:43:31,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:36,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:41,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:42,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:43:42,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:43,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:44,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:43:44,127 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:43:44,127 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:43:44,129 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:43:44,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:44,937 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:43:45,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:45,937 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:43:46,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:47,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:47,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:52,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:57,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:43:58,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:43:58,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:43:59,495 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:43:59,604 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:00,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:44:00,303 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:44:00,303 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:44:00,304 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:44:00,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:00,967 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:44:01,604 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:01,968 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:44:02,604 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:03,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:44:03,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:04,604 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:05,604 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:08,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:44:13,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:44:18,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:44:20,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:44:20,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:21,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:21,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:44:21,519 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:44:21,519 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:44:21,521 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:44:22,014 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:44:22,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:23,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:24,016 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:44:24,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:24,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:44:25,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:26,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:27,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:28,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:29,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:29,496 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:44:30,045 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:44:30,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:31,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:32,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:33,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:34,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:35,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:44:35,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:36,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:37,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:38,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:40,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:44:40,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:41,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:42,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:44:42,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:44:42,459 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:44:42,475 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:44:42,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:43,065 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:44:43,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:44,066 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:44:44,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:44:45,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:44:50,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:44:55,476 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:44:59,498 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:45:00,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:45:05,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:45:06,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:45:06,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:07,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:08,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:45:08,226 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:45:08,226 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:45:08,227 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:45:08,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:09,086 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:45:10,087 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:45:11,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:45:16,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:45:21,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:45:23,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:45:23,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:24,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:24,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:45:24,607 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:45:24,607 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:45:24,623 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:45:25,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:25,099 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:45:26,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:26,100 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:45:26,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:45:29,499 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:45:32,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:45:32,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:45:32,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:33,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:34,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:45:34,203 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:45:34,203 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:45:34,205 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:45:34,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:35,109 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:45:35,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:36,126 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:45:38,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:45:43,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:45:46,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:45:46,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:47,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:47,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:45:47,607 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:45:47,607 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:45:47,608 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:45:48,137 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:45:48,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:48,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:45:49,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:50,139 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:45:50,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:51,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:52,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:53,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:54,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:45:54,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:55,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:56,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:57,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:58,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:59,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:45:59,500 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:45:59,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:00,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:01,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:46:01,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:01,243 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:46:01,243 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:46:01,245 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:46:02,165 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:46:02,165 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:46:02,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:03,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:04,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:05,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:05,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:06,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:11,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:12,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:46:12,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:13,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:13,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:46:13,983 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:46:13,983 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:46:13,985 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:46:14,177 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:46:14,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:16,179 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:46:16,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:21,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:26,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:29,502 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:46:29,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:46:29,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:30,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:31,230 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:46:31,231 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:46:31,231 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:46:31,232 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:46:31,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:32,194 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:46:32,194 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:46:32,232 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:32,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:33,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:34,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:35,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:37,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:41,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:46:41,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:42,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:42,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:46:42,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:46:42,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:46:42,720 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:46:43,204 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:46:43,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:43,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:44,205 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:46:48,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:52,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:46:52,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:53,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:53,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:54,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:46:54,416 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:46:54,417 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:46:54,418 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:46:54,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:55,219 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:46:55,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:56,220 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:46:56,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:57,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:58,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:46:59,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:46:59,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:46:59,928 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:00,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:04,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:47:09,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:47:14,505 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:47:15,872 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:47:15,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:16,873 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:17,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:47:17,402 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:47:17,403 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:47:17,404 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:47:17,873 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:18,253 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:47:18,254 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:47:18,873 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:19,873 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:20,255 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:47:20,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:47:20,873 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:25,873 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:47:29,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:47:29,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:29,504 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:47:30,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:30,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:47:30,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:47:30,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:47:30,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:47:30,929 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:47:31,267 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:47:31,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:32,268 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:47:32,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:33,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:34,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:35,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:36,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:36,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:47:37,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:38,417 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:39,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:40,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:41,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:41,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:47:42,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:46,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:47:50,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:47:50,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:51,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:52,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:47:52,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:47:52,059 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:47:52,059 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:47:52,061 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:47:52,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:47:52,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:53,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:54,289 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:47:54,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:47:57,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:47:59,506 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:48:02,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:48:07,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:48:12,508 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:48:17,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:48:17,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:48:18,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:48:18,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:48:18,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:48:18,723 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:48:18,723 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:48:18,725 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:48:19,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:48:19,346 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:48:20,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:48:20,347 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:48:21,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:48:23,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:48:28,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:48:29,508 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:48:34,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:48:39,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:48:40,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:48:40,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:48:41,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:48:41,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:48:41,591 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:48:41,591 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:48:41,592 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:48:42,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:48:42,366 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:48:43,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:48:44,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:48:44,368 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:48:44,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:48:49,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:48:54,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:48:59,508 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:49:00,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:49:05,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:49:10,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:49:13,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:49:13,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:14,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:15,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:49:15,191 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:49:15,191 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:49:15,205 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:49:15,460 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:49:15,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:16,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:49:16,461 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:49:16,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:17,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:18,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:21,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:49:26,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:49:29,511 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:49:32,513 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:49:34,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:49:34,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:35,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:35,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:49:35,991 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:49:35,991 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:49:35,993 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:49:36,499 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:49:36,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:37,500 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:49:37,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:37,993 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:49:38,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:39,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:40,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:41,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:42,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:43,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:49:47,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:49:47,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:48,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:48,763 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:49:49,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:49:49,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:49:49,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:49:49,341 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:49:49,512 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:49:49,513 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:49:49,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:50,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:51,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:52,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:53,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:54,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:54,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:49:55,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:56,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:57,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:58,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:49:59,514 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:49:59,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:00,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:00,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:01,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:02,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:05,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:10,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:50:10,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:11,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:11,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:11,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:50:11,515 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:50:11,515 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:50:11,517 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:50:11,532 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:50:12,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:13,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:13,534 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:50:14,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:15,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:16,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:17,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:17,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:21,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:22,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:22,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:23,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:50:23,327 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:50:23,327 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:50:23,329 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:50:23,545 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:50:23,545 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:50:23,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:24,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:28,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:29,514 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:50:33,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:37,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:50:37,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:38,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:38,433 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:50:38,433 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:50:38,434 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:50:38,435 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:50:38,559 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:50:39,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:39,435 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:39,560 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:50:40,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:41,072 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:44,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:49,436 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:54,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:56,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:50:56,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:57,537 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:58,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:50:58,079 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:50:58,079 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:50:58,081 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:50:58,537 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:50:58,578 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:50:59,515 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:50:59,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:50:59,579 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:51:04,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:51:09,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:51:14,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:51:19,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:51:23,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:51:23,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:51:24,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:51:24,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:51:25,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:51:25,395 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:51:25,395 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:51:25,396 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:51:25,600 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:51:25,600 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:51:25,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:51:26,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:51:27,602 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:51:27,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:51:28,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:51:29,517 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:51:30,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:51:35,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:51:40,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:51:45,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:51:46,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:51:46,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:51:47,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:51:48,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:51:48,299 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:51:48,299 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:51:48,300 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:51:48,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:51:48,620 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:51:49,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:51:49,621 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:51:50,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:51:51,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:51:56,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:51:59,520 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:52:01,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:06,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:07,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:52:07,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:08,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:08,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:52:08,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:52:08,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:52:08,940 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:52:09,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:09,639 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:52:09,639 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:52:10,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:11,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:11,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:12,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:16,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:17,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:17,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:18,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:52:18,059 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:52:18,059 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:52:18,061 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:52:18,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:18,647 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:52:19,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:19,648 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:52:23,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:28,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:29,519 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:52:29,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:52:29,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:30,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:31,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:52:31,243 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:52:31,243 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:52:31,245 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:52:31,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:31,659 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:52:31,659 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:52:32,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:33,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:33,560 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:34,560 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:35,560 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:36,560 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:37,560 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:38,560 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:42,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:43,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:43,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:43,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:52:43,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:52:43,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:52:43,812 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:52:44,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:44,672 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:52:45,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:45,673 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:52:46,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:47,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:48,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:52:49,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:54,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:59,367 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:52:59,520 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:53:02,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:53:02,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:03,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:03,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:53:03,475 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:53:03,475 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:53:03,477 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:53:03,688 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:53:04,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:04,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:53:05,148 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:05,690 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:53:06,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:07,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:08,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:10,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:53:12,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:13,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:14,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:53:14,363 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:53:14,363 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:53:14,365 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:53:14,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:14,698 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:53:15,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:53:15,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:15,699 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:53:16,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:17,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:18,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:19,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:20,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:53:20,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:21,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:25,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:53:29,523 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:53:29,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:53:29,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:30,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:30,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:53:31,344 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:53:31,344 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:53:31,345 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:53:31,346 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:53:31,713 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:53:31,714 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:53:31,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:32,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:36,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:53:41,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:53:45,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:53:45,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:46,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:46,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:53:46,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:53:46,495 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:53:46,495 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:53:46,496 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:53:46,740 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:53:47,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:47,740 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:53:48,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:49,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:50,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:53:52,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:53:57,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:53:59,523 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:54:02,524 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:54:04,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:54:04,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:05,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:05,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:54:05,807 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:54:05,807 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:54:05,809 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:54:06,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:06,758 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:54:07,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:07,759 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:54:07,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:54:08,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:09,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:10,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:11,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:12,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:13,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:13,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:54:14,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:15,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:19,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:54:24,108 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:54:29,109 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:54:29,524 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:54:34,525 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:54:39,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:54:43,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:54:43,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:44,236 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:44,960 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:54:45,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:54:45,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:54:45,003 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:54:45,005 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:54:45,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:45,823 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:54:45,823 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:54:46,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:47,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:48,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:49,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:50,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:54:50,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:54:55,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:54:59,527 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:55:00,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:01,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:55:01,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:02,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:02,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:55:02,903 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:55:02,903 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:55:02,905 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:55:03,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:03,841 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:55:03,841 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:55:04,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:05,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:05,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:06,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:11,509 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:13,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:55:13,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:14,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:14,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:55:14,891 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:55:14,891 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:55:14,893 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:55:15,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:15,867 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:55:15,867 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:55:16,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:16,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:17,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:18,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:19,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:20,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:21,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:22,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:22,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:23,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:24,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:25,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:26,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:27,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:27,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:28,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:29,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:29,527 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:55:30,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:31,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:32,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:32,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:33,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:34,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:55:34,083 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:55:34,083 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:55:34,085 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:55:34,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:34,885 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:55:35,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:35,886 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:55:36,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:37,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:38,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:38,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:39,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:40,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:41,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:42,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:43,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:43,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:44,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:45,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:48,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:53,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:55,332 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:55:55,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:56,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:56,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:55:56,779 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:55:56,779 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:55:56,780 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:55:56,920 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:55:57,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:55:57,921 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:55:58,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:55:59,528 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:56:04,529 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:56:09,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:56:14,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:56:19,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:56:24,531 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:56:24,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:56:24,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:25,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:26,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:56:26,411 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:56:26,411 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:56:26,413 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:56:26,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:26,945 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:56:27,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:27,946 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:56:28,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:29,548 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:56:29,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:56:29,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:30,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:31,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:34,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:56:39,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:56:44,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:56:45,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:56:45,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:46,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:46,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:56:46,859 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:56:46,859 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:56:46,861 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:56:46,975 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:56:47,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:47,976 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:56:48,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:49,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:50,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:56:50,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:51,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:52,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:53,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:54,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:55,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:56,477 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:56:56,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:57,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:58,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:59,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:56:59,530 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:57:00,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:01,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:57:01,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:02,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:57:02,058 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:57:02,058 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:57:02,060 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:57:02,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:02,991 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:57:03,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:03,992 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:57:07,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:57:11,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:57:11,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:12,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:12,751 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:57:13,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:57:13,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:57:13,415 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:57:13,416 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:57:13,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:14,002 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:57:14,002 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:57:14,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:15,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:16,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:17,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:18,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:57:23,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:57:28,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:57:29,532 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:57:34,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:57:35,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:57:35,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:36,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:37,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:57:37,079 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:57:37,079 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:57:37,081 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:57:37,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:38,022 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:57:38,023 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:57:38,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:39,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:40,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:57:40,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:45,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:57:50,384 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:57:55,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:57:56,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:57:56,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:57,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:57,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:57:57,987 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:57:57,987 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:57:57,989 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:57:58,041 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:57:58,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:59,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:57:59,533 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:58:00,042 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:58:00,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:00,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:58:01,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:02,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:03,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:04,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:05,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:06,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:58:06,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:07,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:08,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:09,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:10,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:11,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:12,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:58:12,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:13,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:14,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:15,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:17,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:58:18,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:19,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:19,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:58:19,723 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:58:19,723 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:58:19,724 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:58:20,061 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:58:20,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:21,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:22,063 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:58:22,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:22,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:58:23,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:24,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:25,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:26,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:27,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:28,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:58:28,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:29,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:29,536 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:58:30,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:31,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:32,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:33,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:34,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:58:34,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:35,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:37,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:38,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:39,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:58:39,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:58:39,643 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:58:39,643 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:58:39,645 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:58:39,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:40,079 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:58:40,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:41,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:42,081 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:58:42,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:43,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:44,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:58:44,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:45,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:49,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:58:51,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:58:51,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:52,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:52,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:58:52,499 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:58:52,499 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:58:52,500 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:58:53,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:53,091 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:58:54,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:54,092 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:58:54,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:58:55,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:56,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:57,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:58,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:59,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:58:59,536 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:58:59,537 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:59:00,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:01,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:02,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:03,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:04,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:05,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:59:09,949 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:59:09,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:10,949 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:10,991 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:59:11,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:59:11,647 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:59:11,647 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:59:11,648 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:59:11,949 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:12,110 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:59:12,949 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:13,949 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:14,112 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:59:14,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:15,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:16,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:16,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:59:17,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:21,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:59:26,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:59:28,265 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 13:59:28,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:29,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:29,537 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 13:59:29,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:59:29,671 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:59:29,671 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:59:29,673 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:59:30,127 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:59:30,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:31,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:32,129 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:59:32,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:32,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:59:33,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:34,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:35,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:36,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:37,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:38,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:59:38,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:39,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:40,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:41,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:42,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:43,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:44,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:59:44,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:45,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:46,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:47,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:48,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:50,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:59:51,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:52,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:52,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 13:59:52,679 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 13:59:52,680 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 13:59:52,681 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 13:59:53,169 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 13:59:53,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:54,170 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 13:59:54,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:55,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 13:59:55,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:56,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:57,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:58,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:59,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 13:59:59,541 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:00:00,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:00,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:00:01,299 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:02,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:03,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:04,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:05,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:06,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:00:06,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:07,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:08,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:09,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:10,300 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:11,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:00:11,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:12,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:12,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:00:12,522 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:00:12,522 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:00:12,523 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:00:13,188 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:00:13,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:14,189 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:00:14,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:15,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:16,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:00:16,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:17,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:18,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:19,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:20,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:21,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:22,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:00:22,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:23,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:28,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:00:29,540 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:00:32,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:00:32,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:33,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:33,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:00:33,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:00:33,987 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:00:33,987 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:00:33,988 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:00:34,209 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:00:34,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:36,211 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:00:38,989 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:00:43,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:00:48,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:00:51,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:00:51,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:52,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:52,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:00:52,575 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:00:52,575 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:00:52,577 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:00:53,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:53,231 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:00:54,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:54,232 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:00:54,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:00:55,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:56,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:57,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:58,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:59,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:00:59,541 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:01:00,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:01,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:02,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:03,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:01:03,075 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:01:03,075 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:01:03,076 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:01:03,241 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:01:03,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:04,242 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:01:04,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:05,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:06,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:06,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:07,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:08,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:09,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:10,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:11,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:11,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:12,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:13,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:14,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:15,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:16,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:17,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:17,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:18,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:19,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:20,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:21,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:22,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:23,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:23,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:01:23,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:01:23,459 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:01:23,462 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:01:23,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:24,286 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:01:24,287 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:01:24,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:28,463 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:29,545 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:01:33,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:38,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:42,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:01:42,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:43,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:43,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:44,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:01:44,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:01:44,003 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:01:44,005 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:01:44,305 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:01:44,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:45,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:46,307 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:01:46,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:01:49,005 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:54,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:59,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:01:59,544 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:02:04,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:02:09,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:02:14,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:02:16,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:02:16,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:17,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:17,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:02:17,647 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:02:17,648 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:02:17,649 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:02:18,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:18,350 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:02:19,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:19,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:02:20,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:20,352 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:02:21,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:22,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:25,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:02:29,546 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:02:30,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:02:35,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:02:40,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:02:45,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:02:45,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:46,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:46,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:02:47,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:02:47,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:02:47,003 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:02:47,005 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:02:47,392 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:02:47,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:48,393 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:02:48,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:49,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:50,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:51,473 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:52,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:52,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:02:53,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:56,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:57,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:57,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:02:57,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:02:57,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:02:57,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:02:57,797 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:02:58,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:58,404 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:02:59,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:02:59,547 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:03:00,411 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:03:03,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:03:08,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:03:09,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:03:09,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:10,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:11,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:03:11,147 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:03:11,147 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:03:11,149 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:03:11,421 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:03:11,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:12,422 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:03:12,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:13,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:14,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:03:19,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:03:24,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:03:29,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:03:29,550 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:03:34,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:03:34,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:34,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:03:35,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:35,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:03:35,691 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:03:35,691 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:03:35,693 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:03:36,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:36,443 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:03:37,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:38,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:38,444 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:03:39,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:40,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:40,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:03:41,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:43,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:44,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:45,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:03:45,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:03:45,175 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:03:45,177 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:03:45,452 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:03:45,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:46,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:03:46,453 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:03:46,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:47,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:48,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:49,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:50,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:03:51,340 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:03:56,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:03:59,550 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:04:01,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:04,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:04:04,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:05,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:05,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:04:05,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:04:05,635 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:04:05,637 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:04:06,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:06,471 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:04:06,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:08,472 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:04:11,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:16,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:21,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:26,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:29,551 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:04:31,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:04:31,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:31,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:32,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:33,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:04:33,475 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:04:33,475 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:04:33,476 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:04:33,491 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:04:33,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:34,492 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:04:34,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:35,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:36,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:37,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:37,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:38,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:39,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:40,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:41,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:43,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:48,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:49,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:04:49,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:50,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:50,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:04:50,899 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:04:50,899 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:04:50,901 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:04:51,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:51,508 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:04:52,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:52,509 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:04:53,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:53,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:54,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:04:59,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:04:59,554 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:05:04,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:05:08,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:05:08,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:09,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:09,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:05:10,230 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:05:10,231 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:05:10,231 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:05:10,234 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:05:10,524 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:05:10,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:11,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:12,526 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:05:12,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:13,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:14,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:15,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:15,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:05:16,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:17,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:20,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:05:24,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:05:24,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:25,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:25,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:05:26,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:05:26,423 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:05:26,423 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:05:26,425 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:05:26,556 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:05:26,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:28,558 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:05:29,554 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:05:31,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:05:36,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:05:41,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:05:46,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:05:48,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:05:48,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:49,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:50,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:05:50,027 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:05:50,027 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:05:50,029 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:05:50,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:50,592 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:05:51,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:52,029 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:05:52,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:52,594 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:05:53,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:54,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:05:57,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:05:59,555 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:06:02,556 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:06:07,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:06:12,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:06:12,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:06:12,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:13,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:14,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:06:14,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:06:14,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:06:14,097 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:06:14,613 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:06:14,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:15,614 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:06:15,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:16,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:17,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:06:17,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:18,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:19,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:20,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:22,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:06:27,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:06:29,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:06:29,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:29,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:06:30,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:30,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:06:30,567 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:06:30,567 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:06:30,569 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:06:30,628 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:06:31,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:31,629 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:06:32,040 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:33,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:33,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:06:38,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:06:42,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:06:42,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:43,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:43,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:06:44,276 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:06:44,277 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:06:44,277 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:06:44,278 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:06:44,641 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:06:44,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:45,642 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:06:45,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:46,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:47,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:06:49,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:06:54,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:06:59,558 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:07:00,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:05,560 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:10,560 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:15,561 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:20,561 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:21,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:07:21,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:22,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:23,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:07:23,155 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:07:23,155 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:07:23,157 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:07:23,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:23,685 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:07:23,685 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:07:24,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:25,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:26,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:26,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:27,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:28,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:29,559 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:07:31,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:31,560 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:32,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:32,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:07:32,723 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:07:32,723 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:07:32,724 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:07:33,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:33,695 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:07:33,695 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:07:36,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:41,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:46,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:51,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:56,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:07:58,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:07:58,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:59,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:07:59,561 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:08:00,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:08:00,067 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:08:00,067 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:08:00,068 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:08:00,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:00,734 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:08:01,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:01,735 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:08:02,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:02,399 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:07,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:12,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:17,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:17,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:08:17,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:18,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:19,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:08:19,543 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:08:19,543 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:08:19,544 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:08:19,765 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:08:19,765 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:08:19,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:20,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:21,767 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:08:21,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:22,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:22,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:23,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:27,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:29,564 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:08:33,565 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:36,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:08:36,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:37,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:37,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:08:37,751 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:08:37,751 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:08:37,753 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:08:37,785 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:08:38,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:38,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:39,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:39,787 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:08:43,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:48,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:52,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:08:52,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:53,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:08:53,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:53,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:08:53,895 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:08:53,895 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:08:53,897 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:08:54,827 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:08:55,841 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:08:58,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:08:59,564 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:09:04,565 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:09:04,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:09:04,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:09:05,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:09:06,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:09:06,187 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:09:06,187 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:09:06,190 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:09:06,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:09:06,852 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:09:07,853 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:09:10,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:09:15,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:09:20,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:09:25,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:09:29,565 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:09:30,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:09:34,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:09:34,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:09:35,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:09:35,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:09:35,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:09:35,895 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:09:35,895 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:09:35,896 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:09:36,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:09:36,876 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:09:37,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:09:37,877 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:09:38,271 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:09:40,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:09:45,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:09:50,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:09:55,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:09:57,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:09:57,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:09:58,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:09:59,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:09:59,135 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:09:59,135 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:09:59,136 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:09:59,568 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:09:59,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:09:59,897 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:09:59,897 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:10:00,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:01,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:01,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:02,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:03,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:06,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:11,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:12,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:10:12,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:13,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:13,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:10:13,965 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:10:13,965 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:10:13,966 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:10:14,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:14,910 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:10:15,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:15,911 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:10:16,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:16,967 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:17,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:18,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:19,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:20,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:21,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:22,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:22,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:27,307 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:29,568 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:10:30,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:10:30,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:31,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:31,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:10:31,891 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:10:31,891 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:10:31,894 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:10:31,939 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:10:32,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:32,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:33,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:33,941 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:10:37,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:42,895 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:44,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:10:44,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:45,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:46,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:10:46,059 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:10:46,059 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:10:46,061 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:10:46,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:46,955 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:10:47,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:47,956 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:10:48,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:48,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:49,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:50,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:51,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:52,391 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:53,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:53,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:57,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:58,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:58,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:10:59,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:10:59,123 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:10:59,123 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:10:59,125 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:10:59,569 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:10:59,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:10:59,998 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:10:59,998 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:11:00,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:01,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:02,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:04,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:11:09,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:11:14,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:11:19,572 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:11:21,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:11:21,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:22,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:23,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:11:23,271 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:11:23,271 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:11:23,274 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:11:23,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:24,032 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:11:24,033 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:11:24,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:25,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:11:25,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:26,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:29,572 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:11:30,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:11:35,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:11:40,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:11:45,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:11:50,109 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:11:50,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:51,109 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:51,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:11:51,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:11:51,615 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:11:51,615 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:11:51,617 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:11:52,057 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:11:52,109 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:53,109 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:54,059 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:11:54,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:55,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:56,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:57,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:57,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:11:58,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:11:59,572 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:12:01,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:02,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:02,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:12:03,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:12:03,167 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:12:03,167 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:12:03,168 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:12:03,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:04,083 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:12:04,083 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:12:04,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:05,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:06,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:08,479 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:12:10,507 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:11,508 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:12,210 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:12:12,211 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:12:12,211 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:12:12,213 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:12:12,508 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:13,105 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:12:13,508 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:14,106 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:12:14,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:12:14,508 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:19,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:12:22,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:12:22,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:23,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:24,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:12:24,179 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:12:24,179 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:12:24,182 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:12:24,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:25,117 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:12:25,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:12:26,118 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:12:29,573 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:12:30,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:12:35,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:12:40,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:12:45,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:12:49,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:12:49,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:50,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:50,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:12:51,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:12:51,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:12:51,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:12:51,211 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:12:51,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:52,158 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:12:52,158 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:12:52,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:53,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:54,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:12:56,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:12:59,574 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:13:01,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:13:06,576 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:13:08,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:13:08,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:09,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:09,920 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:13:09,922 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:13:09,922 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:13:09,923 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:13:10,175 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:13:10,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:11,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:13:12,177 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:13:16,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:13:21,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:13:24,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:13:24,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:25,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:26,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:13:26,531 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:13:26,531 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:13:26,533 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:13:26,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:27,205 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:13:27,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:13:27,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:28,206 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:13:28,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:29,576 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:13:29,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:30,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:31,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:32,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:32,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:13:33,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:34,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:35,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:36,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:37,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:38,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:13:38,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:39,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:40,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:41,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:42,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:43,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:13:43,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:44,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:45,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:46,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:13:46,479 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:13:46,479 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:13:46,481 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:13:46,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:47,226 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:13:47,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:48,227 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:13:48,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:49,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:13:54,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:13:55,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:13:55,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:56,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:57,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:13:57,051 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:13:57,051 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:13:57,053 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:13:57,235 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:13:57,664 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:13:58,236 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:13:59,577 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:13:59,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:04,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:09,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:10,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:14:10,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:11,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:12,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:14:12,187 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:14:12,187 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:14:12,189 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:14:12,249 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:14:12,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:13,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:14,251 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:14:14,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:15,189 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:20,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:25,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:29,578 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:14:30,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:35,580 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:35,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:14:35,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:36,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:37,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:14:37,283 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:14:37,283 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:14:37,284 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:14:37,287 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:14:37,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:38,288 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:14:38,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:39,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:40,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:40,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:41,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:42,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:43,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:44,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:45,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:46,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:46,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:47,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:48,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:49,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:50,907 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:51,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:52,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:52,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:53,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:54,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:55,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:56,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:57,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:14:57,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:58,908 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:14:59,596 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:15:01,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:02,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:02,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:15:02,451 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:15:02,451 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:15:02,453 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:15:03,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:03,310 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:15:03,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:15:04,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:04,310 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:15:05,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:06,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:07,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:08,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:09,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:09,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:15:10,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:11,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:12,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:13,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:14,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:14,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:15:14,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:15:14,547 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:15:14,547 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:15:14,549 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:15:15,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:15,321 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:15:16,322 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:15:19,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:15:24,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:15:29,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:15:29,581 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:15:34,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:15:35,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:15:35,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:36,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:37,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:15:37,307 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:15:37,307 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:15:37,309 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:15:37,341 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:15:37,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:38,342 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:15:38,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:39,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:40,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:15:40,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:41,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:42,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:45,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:15:49,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:15:49,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:50,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:50,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:15:50,727 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:15:50,727 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:15:50,728 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:15:51,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:51,364 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:15:51,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:15:52,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:52,365 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:15:53,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:15:56,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:15:59,582 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:16:02,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:16:07,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:16:12,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:16:14,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:16:14,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:15,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:16,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:16:16,362 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:16:16,362 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:16:16,365 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:16:16,387 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:16:16,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:18,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:16:18,389 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:16:23,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:16:28,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:16:29,585 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:16:33,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:16:38,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:16:43,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:16:44,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:16:44,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:45,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:46,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:16:46,039 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:16:46,039 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:16:46,040 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:16:46,409 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:16:46,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:47,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:48,410 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:16:49,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:16:54,041 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:16:55,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:16:55,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:56,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:56,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:16:56,827 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:16:56,827 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:16:56,830 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:16:57,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:57,418 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:16:58,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:58,419 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:16:59,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:16:59,585 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:16:59,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:00,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:01,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:05,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:10,135 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:15,135 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:20,136 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:25,136 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:29,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:17:30,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:33,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:17:34,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:34,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:35,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:17:35,391 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:17:35,391 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:17:35,407 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:17:35,446 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:17:35,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:36,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:36,447 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:17:36,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:37,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:38,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:39,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:41,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:46,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:51,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:53,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:17:53,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:54,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:54,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:17:54,703 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:17:54,703 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:17:54,705 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:17:55,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:55,507 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:17:56,239 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:17:56,508 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:17:57,705 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:17:59,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:18:03,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:18:08,589 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:18:08,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:18:08,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:09,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:10,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:18:10,634 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:18:10,634 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:18:10,635 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:18:10,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:11,523 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:18:11,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:12,524 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:18:12,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:13,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:18:13,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:14,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:15,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:18,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:18:20,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:21,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:22,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:18:22,171 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:18:22,171 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:18:22,172 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:18:22,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:22,536 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:18:23,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:24,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:18:24,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:24,538 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:18:25,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:26,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:27,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:28,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:29,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:29,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:18:29,589 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:18:30,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:31,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:32,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:33,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:34,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:35,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:18:35,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:36,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:37,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:38,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:39,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:40,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:18:40,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:41,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:42,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:43,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:44,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:45,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:18:45,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:45,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:18:45,887 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:18:45,887 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:18:45,888 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:18:46,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:46,573 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:18:47,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:48,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:48,575 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:18:49,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:50,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:18:50,428 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:51,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:52,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:53,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:54,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:55,427 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:18:55,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:18:59,590 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:19:00,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:05,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:06,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:19:06,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:07,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:08,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:19:08,347 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:19:08,347 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:19:08,348 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:19:08,592 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:19:08,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:10,594 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:19:11,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:16,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:21,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:26,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:19:26,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:27,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:27,079 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:27,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:19:27,475 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:19:27,475 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:19:27,477 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:19:27,624 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:19:28,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:28,625 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:19:29,591 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:19:32,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:35,244 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:19:35,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:36,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:36,616 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:19:36,616 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:19:36,616 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:19:36,618 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:19:36,647 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:19:37,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:37,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:38,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:38,649 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:19:39,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:40,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:41,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:42,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:43,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:43,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:44,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:19:48,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:53,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:58,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:19:59,595 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:19:59,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:19:59,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:00,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:01,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:20:01,375 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:20:01,375 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:20:01,391 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:20:01,667 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:20:01,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:02,668 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:20:02,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:03,392 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:20:03,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:04,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:05,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:06,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:07,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:08,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:20:08,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:09,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:14,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:20:19,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:20:20,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:20:20,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:21,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:21,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:20:21,759 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:20:21,759 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:20:21,761 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:20:22,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:22,703 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:20:23,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:23,704 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:20:24,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:25,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:20:25,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:26,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:29,595 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:20:30,596 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:20:35,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:20:38,319 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:20:38,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:39,319 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:39,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:20:39,755 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:20:39,755 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:20:39,756 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:20:40,319 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:40,735 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:20:40,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:20:41,319 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:41,736 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:20:42,319 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:43,319 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:46,319 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:20:51,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:20:51,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:52,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:52,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:20:52,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:20:52,683 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:20:52,683 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:20:52,685 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:20:52,745 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:20:53,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:53,746 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:20:54,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:55,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:56,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:57,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:58,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:58,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:20:59,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:20:59,597 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:21:00,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:01,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:04,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:21:09,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:21:13,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:21:13,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:14,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:14,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:21:14,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:21:14,563 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:21:14,563 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:21:14,565 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:21:14,762 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:21:15,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:15,778 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:21:16,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:17,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:18,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:19,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:20,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:20,133 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:21:21,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:22,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:23,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:24,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:25,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:26,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:21:29,600 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:21:31,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:21:31,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:21:31,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:32,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:33,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:21:33,199 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:21:33,199 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:21:33,200 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:21:33,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:33,809 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:21:33,809 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:21:34,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:35,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:36,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:37,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:37,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:21:38,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:42,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:21:47,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:21:52,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:21:52,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:21:52,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:53,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:54,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:21:54,427 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:21:54,427 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:21:54,429 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:21:54,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:54,826 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:21:55,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:55,858 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:21:56,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:21:58,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:21:59,599 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:22:03,600 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:22:08,601 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:22:10,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:22:10,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:11,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:11,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:22:11,531 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:22:11,531 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:22:11,533 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:22:11,870 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:22:11,870 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:22:12,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:13,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:14,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:14,533 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:22:15,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:16,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:17,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:18,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:19,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:20,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:22:20,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:21,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:22,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:23,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:24,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:24,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:22:24,527 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:22:24,527 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:22:24,528 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:22:24,887 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:22:25,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:22:25,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:25,902 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:22:26,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:27,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:29,601 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:22:30,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:22:35,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:22:40,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:22:45,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:22:48,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:22:48,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:49,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:49,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:22:49,919 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:22:49,920 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:22:49,921 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:22:49,922 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:22:50,313 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:50,921 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:22:51,924 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:22:55,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:22:57,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:22:57,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:58,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:59,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:22:59,451 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:22:59,451 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:22:59,452 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:22:59,605 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:22:59,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:22:59,931 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:22:59,931 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:23:01,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:23:06,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:23:11,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:23:16,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:23:21,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:23:26,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:23:26,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:27,127 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:27,171 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:23:27,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:23:27,511 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:23:27,511 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:23:27,513 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:23:27,966 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:23:27,966 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:23:28,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:29,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:29,603 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:23:30,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:31,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:32,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:33,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:33,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:23:34,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:35,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:36,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:37,128 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:38,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:39,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:39,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:23:40,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:41,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:42,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:43,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:44,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:44,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:23:45,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:46,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:47,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:47,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:23:47,759 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:23:47,759 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:23:47,760 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:23:47,982 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:23:48,129 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:23:49,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:23:49,984 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:23:54,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:23:59,605 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:24:00,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:24:03,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:24:03,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:04,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:04,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:24:04,667 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:24:04,667 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:24:04,669 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:24:04,997 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:24:05,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:05,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:24:05,998 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:24:06,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:07,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:08,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:09,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:10,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:11,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:11,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:24:12,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:13,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:14,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:15,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:16,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:17,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:17,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:24:18,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:19,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:20,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:22,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:24:25,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:24:25,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:26,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:26,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:24:26,763 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:24:26,763 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:24:26,765 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:24:27,031 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:24:27,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:27,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:24:28,032 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:24:29,606 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:24:33,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:24:38,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:24:43,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:24:43,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:44,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:44,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:24:44,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:24:44,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:24:44,815 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:24:44,816 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:24:45,061 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:24:45,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:46,062 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:24:46,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:24:49,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:24:54,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:24:59,609 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:25:00,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:01,109 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:25:01,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:02,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:02,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:25:02,751 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:25:02,751 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:25:02,752 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:25:03,091 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:25:03,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:04,092 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:25:04,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:05,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:10,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:15,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:18,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:25:18,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:19,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:19,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:25:19,567 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:25:19,567 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:25:19,568 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:25:20,107 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:25:20,107 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:25:20,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:21,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:21,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:22,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:26,569 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:29,609 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:25:31,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:32,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:25:32,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:33,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:33,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:25:33,739 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:25:33,739 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:25:33,741 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:25:34,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:34,119 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:25:35,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:36,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:36,138 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:25:36,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:37,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:42,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:43,252 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:25:43,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:44,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:44,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:25:44,614 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:25:44,615 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:25:44,616 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:25:45,146 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:25:45,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:46,147 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:25:46,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:47,617 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:52,617 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:53,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:25:53,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:54,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:55,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:25:55,115 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:25:55,116 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:25:55,117 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:25:55,156 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:25:55,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:56,157 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:25:56,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:25:58,117 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:25:59,610 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:26:03,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:26:08,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:26:09,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:26:09,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:10,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:10,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:26:10,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:26:10,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:26:10,720 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:26:11,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:11,183 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:26:12,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:12,184 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:26:13,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:13,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:26:14,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:15,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:16,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:17,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:19,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:26:24,139 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:26:26,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:26:26,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:27,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:27,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:26:27,707 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:26:27,707 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:26:27,709 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:26:28,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:28,199 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:26:29,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:29,629 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:26:29,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:26:30,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:30,200 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:26:31,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:32,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:33,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:34,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:35,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:26:35,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:26:40,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:26:45,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:26:50,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:26:55,051 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:26:59,612 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:27:00,276 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:27:00,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:00,319 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:27:01,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:01,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:27:01,663 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:27:01,663 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:27:01,665 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:27:02,239 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:27:02,239 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:27:02,276 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:03,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:04,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:05,276 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:06,276 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:27:06,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:07,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:11,276 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:27:12,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:27:12,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:13,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:13,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:27:13,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:27:13,815 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:27:13,817 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:27:14,250 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:27:14,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:16,252 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:27:16,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:27:20,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:27:21,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:21,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:22,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:27:22,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:27:22,471 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:27:22,471 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:27:22,473 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:27:22,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:23,259 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:27:23,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:24,260 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:27:24,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:25,973 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:26,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:27,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:27,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:27:28,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:29,614 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:27:29,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:30,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:33,972 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:27:38,973 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:27:43,973 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:27:44,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:27:44,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:45,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:45,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:27:45,647 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:27:45,648 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:27:45,651 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:27:46,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:46,315 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:27:46,316 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:27:47,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:48,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:27:49,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:27:54,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:27:59,617 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:28:00,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:05,619 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:05,864 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:28:05,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:28:06,864 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:28:07,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:28:07,299 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:28:07,299 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:28:07,301 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:28:07,352 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:28:07,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:28:08,353 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:28:08,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:28:11,301 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:16,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:21,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:26,303 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:29,616 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:28:31,617 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:36,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:38,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:28:38,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:28:39,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:28:40,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:28:40,259 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:28:40,260 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:28:40,261 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:28:40,379 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:28:40,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:28:42,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:42,381 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:28:47,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:52,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:57,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:28:57,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:28:57,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:28:58,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:28:59,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:28:59,125 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:28:59,125 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:28:59,127 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:28:59,396 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:28:59,617 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:28:59,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:00,396 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:29:00,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:01,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:02,619 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:29:02,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:03,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:04,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:05,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:06,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:07,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:29:12,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:29:17,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:29:17,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:29:17,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:18,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:19,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:29:19,487 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:29:19,487 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:29:19,488 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:29:19,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:20,414 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:29:20,415 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:29:20,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:21,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:22,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:29:22,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:27,936 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:29:29,619 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:29:33,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:29:38,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:29:41,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:29:41,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:42,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:43,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:29:43,207 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:29:43,207 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:29:43,208 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:29:43,446 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:29:43,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:44,209 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:29:44,447 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:29:44,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:45,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:46,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:47,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:48,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:49,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:49,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:29:50,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:51,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:52,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:53,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:54,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:55,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:29:55,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:56,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:57,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:58,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:29:59,637 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:29:59,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:00,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:01,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:01,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:02,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:03,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:04,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:05,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:06,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:06,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:07,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:08,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:09,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:10,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:11,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:11,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:12,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:13,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:30:13,347 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:30:13,347 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:30:13,349 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:30:13,471 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:30:13,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:14,472 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:30:14,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:15,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:16,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:16,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:17,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:18,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:21,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:26,685 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:29,622 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:30:31,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:30:31,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:31,871 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:32,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:33,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:30:33,275 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:30:33,275 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:30:33,277 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:30:33,488 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:30:33,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:34,489 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:30:34,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:35,829 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:37,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:42,278 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:45,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:30:45,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:46,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:47,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:30:47,127 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:30:47,128 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:30:47,129 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:30:47,512 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:30:47,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:48,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:48,513 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:30:48,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:49,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:50,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:51,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:30:53,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:58,655 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:30:59,623 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:31:04,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:31:09,625 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:31:13,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:31:13,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:31:14,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:31:14,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:31:14,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:31:14,627 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:31:14,627 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:31:14,629 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:31:15,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:31:15,599 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:31:16,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:31:16,600 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:31:17,339 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:31:19,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:31:24,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:31:29,626 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:31:30,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:31:35,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:31:40,628 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:31:45,628 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:31:46,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:31:46,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:31:47,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:31:48,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:31:48,375 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:31:48,375 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:31:48,378 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:31:48,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:31:48,638 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:31:49,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:31:50,640 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:31:51,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:31:56,379 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:31:57,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:31:58,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:31:58,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:31:59,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:31:59,603 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:31:59,603 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:31:59,605 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:31:59,626 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:31:59,648 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:31:59,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:00,649 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:32:00,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:01,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:01,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:02,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:03,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:04,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:06,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:11,964 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:16,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:18,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:32:18,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:19,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:20,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:32:20,027 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:32:20,027 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:32:20,029 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:32:20,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:20,667 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:32:21,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:22,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:22,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:22,669 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:32:23,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:24,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:25,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:26,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:27,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:27,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:28,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:29,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:29,627 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:32:30,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:31,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:32,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:32,628 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:33,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:34,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:35,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:36,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:37,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:38,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:38,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:39,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:40,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:41,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:42,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:43,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:43,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:44,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:45,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:46,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:47,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:48,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:48,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:49,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:50,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:51,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:52,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:53,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:54,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:54,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:55,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:56,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:57,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:58,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:59,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:32:59,086 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:32:59,086 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:32:59,088 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:32:59,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:32:59,630 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:32:59,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:32:59,699 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:33:00,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:00,700 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:33:01,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:02,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:03,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:04,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:05,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:33:05,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:06,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:07,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:08,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:09,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:10,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:33:10,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:11,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:15,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:33:20,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:33:21,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:33:21,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:22,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:22,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:33:22,959 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:33:22,959 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:33:22,961 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:33:23,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:23,720 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:33:24,536 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:24,721 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:33:25,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:33:29,629 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:33:31,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:33:36,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:33:39,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:33:39,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:40,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:41,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:33:41,395 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:33:41,395 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:33:41,397 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:33:41,735 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:33:41,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:42,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:33:42,736 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:33:42,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:43,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:44,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:45,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:46,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:47,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:47,945 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:33:48,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:49,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:50,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:33:52,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:33:57,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:33:59,631 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:33:59,983 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:34:00,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:00,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:01,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:34:01,439 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:34:01,439 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:34:01,440 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:34:01,751 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:34:01,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:02,752 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:34:03,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:34:08,441 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:34:13,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:34:18,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:34:19,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:34:19,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:20,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:20,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:34:20,571 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:34:20,571 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:34:20,573 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:34:20,781 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:34:21,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:22,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:22,783 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:34:23,155 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:23,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:34:24,155 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:25,155 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:26,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:27,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:28,156 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:29,155 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:34:29,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:29,632 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:34:30,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:31,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:34:31,115 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:34:31,115 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:34:31,116 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:34:31,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:31,792 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:34:31,792 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:34:32,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:35,117 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:34:40,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:34:45,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:34:50,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:34:51,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:34:51,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:52,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:52,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:34:52,779 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:34:52,779 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:34:52,781 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:34:52,809 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:34:53,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:53,810 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:34:54,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:55,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:55,781 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:34:56,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:57,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:58,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:59,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:34:59,635 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:35:00,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:35:01,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:06,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:10,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:35:10,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:35:11,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:35:11,655 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:12,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:35:12,079 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:35:12,079 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:35:12,080 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:35:12,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:35:12,842 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:35:13,843 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:35:17,081 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:22,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:25,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:35:25,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:35:26,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:35:26,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:35:26,767 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:35:26,767 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:35:26,768 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:35:26,899 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:35:27,411 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:35:27,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:27,900 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:35:29,634 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:35:33,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:38,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:43,636 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:45,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:35:45,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:35:46,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:35:46,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:35:46,675 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:35:46,675 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:35:46,677 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:35:46,951 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:35:47,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:35:47,952 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:35:48,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:35:48,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:53,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:58,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:35:59,636 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:36:04,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:36:09,637 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:36:14,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:36:19,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:36:23,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:36:23,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:24,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:24,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:36:25,056 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:36:25,057 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:36:25,057 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:36:25,058 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:36:25,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:25,999 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:36:25,999 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:36:26,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:27,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:28,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:29,652 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:36:29,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:30,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:36:30,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:31,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:32,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:33,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:34,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:35,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:36:35,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:40,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:36:41,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:36:41,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:42,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:42,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:36:42,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:36:42,503 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:36:42,505 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:36:43,014 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:36:43,032 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:44,015 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:36:44,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:45,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:46,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:36:46,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:36:51,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:36:56,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:36:59,638 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:37:01,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:37:06,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:37:11,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:37:14,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:37:14,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:15,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:16,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:37:16,411 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:37:16,411 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:37:16,413 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:37:16,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:17,040 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:37:17,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:37:17,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:18,041 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:37:18,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:19,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:20,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:21,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:22,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:22,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:37:23,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:27,685 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:37:29,640 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:37:33,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:37:38,641 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:37:43,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:37:43,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:44,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:44,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:37:45,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:37:45,239 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:37:45,239 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:37:45,241 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:37:45,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:46,063 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:37:46,063 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:37:46,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:47,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:48,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:49,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:37:50,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:37:55,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:37:59,643 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:38:00,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:01,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:38:01,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:02,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:02,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:38:02,907 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:38:02,907 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:38:02,909 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:38:03,080 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:38:03,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:04,081 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:38:04,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:05,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:10,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:15,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:20,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:38:20,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:21,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:21,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:21,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:38:21,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:38:21,587 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:38:21,589 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:38:22,095 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:38:22,095 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:38:22,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:23,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:24,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:25,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:26,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:27,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:27,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:28,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:29,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:29,642 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:38:30,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:31,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:32,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:33,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:38,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:43,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:48,183 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:53,184 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:58,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:38:58,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:38:58,618 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:59,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:38:59,644 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:39:00,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:39:00,311 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:39:00,311 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:39:00,313 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:39:00,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:01,143 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:39:01,575 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:02,144 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:39:03,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:39:08,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:39:13,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:39:18,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:39:22,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:39:22,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:23,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:23,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:39:23,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:39:23,671 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:39:23,671 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:39:23,673 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:39:24,170 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:39:24,170 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:39:24,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:25,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:26,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:27,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:28,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:29,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:39:29,645 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:39:34,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:39:39,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:39:41,977 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:39:42,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:42,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:43,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:39:43,631 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:39:43,631 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:39:43,632 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:39:43,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:39:44,189 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:39:44,190 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:39:45,632 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:39:50,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:39:55,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:39:59,664 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:40:00,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:40:05,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:40:10,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:40:13,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:40:13,706 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:14,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:15,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:40:15,087 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:40:15,087 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:40:15,089 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:40:15,227 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:40:15,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:16,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:40:16,228 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:40:16,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:17,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:18,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:19,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:20,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:21,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:40:21,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:22,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:26,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:40:29,647 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:40:32,648 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:40:37,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:40:42,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:40:42,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:43,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:43,323 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:40:43,535 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:40:43,535 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:40:43,535 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:40:43,537 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:40:44,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:44,293 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:40:44,293 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:40:45,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:46,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:47,280 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:48,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:49,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:49,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:40:53,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:54,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:54,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:40:54,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:40:54,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:40:54,955 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:40:54,956 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:40:55,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:55,335 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:40:56,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:56,336 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:40:57,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:58,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:59,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:40:59,648 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:40:59,649 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:00,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:01,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:04,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:09,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:12,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:41:12,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:13,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:14,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:41:14,135 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:41:14,135 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:41:14,136 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:41:14,364 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:41:14,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:15,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:15,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:16,379 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:41:16,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:17,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:18,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:19,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:20,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:25,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:29,652 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:41:31,653 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:34,171 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:41:34,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:35,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:35,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:41:35,875 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:41:35,875 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:41:35,876 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:41:36,171 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:36,424 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:41:36,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:37,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:38,426 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:41:41,877 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:46,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:48,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:41:48,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:49,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:50,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:41:50,339 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:41:50,339 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:41:50,342 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:41:50,451 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:41:50,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:41:52,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:52,453 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:41:57,343 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:41:59,651 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:42:02,652 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:42:05,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:42:05,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:06,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:06,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:42:06,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:42:06,695 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:42:06,697 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:42:07,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:07,493 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:42:07,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:42:08,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:08,493 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:42:09,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:10,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:13,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:42:15,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:42:15,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:16,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:17,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:42:17,031 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:42:17,031 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:42:17,033 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:42:17,501 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:42:17,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:18,502 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:42:18,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:19,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:42:19,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:20,635 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:24,635 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:42:29,635 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:42:29,652 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:42:30,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:42:30,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:31,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:32,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:42:32,391 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:42:32,391 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:42:32,393 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:42:32,530 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:42:32,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:33,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:34,532 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:42:34,914 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:35,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:42:35,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:36,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:37,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:38,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:39,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:40,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:42:40,916 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:41,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:45,915 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:42:46,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:47,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:48,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:42:48,219 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:42:48,219 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:42:48,221 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:42:48,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:48,607 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:42:49,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:50,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:50,611 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:42:51,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:42:51,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:52,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:53,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:54,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:55,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:56,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:42:56,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:57,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:58,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:59,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:42:59,655 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:43:00,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:01,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:02,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:43:02,460 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:07,459 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:43:09,385 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:43:09,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:10,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:10,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:43:10,763 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:43:10,763 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:43:10,764 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:43:11,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:11,660 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:43:12,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:12,661 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:43:12,765 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:43:13,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:14,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:15,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:16,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:17,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:18,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:18,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:43:19,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:20,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:21,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:22,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:23,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:23,387 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:43:26,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:27,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:28,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:43:28,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:43:28,236 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:43:28,237 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:43:28,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:28,676 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:43:29,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:43:29,655 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:43:30,677 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:43:34,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:43:39,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:43:44,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:43:47,564 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:43:47,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:48,564 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:48,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:43:48,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:43:48,955 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:43:48,956 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:43:49,564 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:49,694 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:43:49,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:43:50,564 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:43:50,695 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:43:54,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:43:59,656 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:44:00,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:05,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:06,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:44:06,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:07,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:07,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:44:07,879 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:44:07,879 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:44:07,881 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:44:08,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:08,710 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:44:10,711 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:44:10,881 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:15,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:20,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:25,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:29,658 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:44:31,659 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:36,660 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:40,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:44:40,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:41,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:41,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:42,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:44:42,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:44:42,175 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:44:42,177 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:44:42,738 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:44:42,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:43,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:44,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:44,739 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:44:47,178 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:52,179 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:55,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:44:55,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:56,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:57,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:44:57,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:44:57,403 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:44:57,403 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:44:57,405 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:44:57,765 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:44:57,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:58,766 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:44:58,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:44:59,665 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:45:02,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:45:07,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:45:09,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:45:09,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:10,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:10,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:45:10,587 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:45:10,587 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:45:10,589 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:45:10,790 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:45:11,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:12,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:12,792 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:45:13,160 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:13,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:45:14,161 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:15,161 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:16,161 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:19,161 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:45:24,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:45:29,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:45:29,660 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:45:34,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:45:37,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:45:37,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:38,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:39,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:45:39,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:45:39,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:45:39,197 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:45:39,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:39,842 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:45:40,197 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:45:40,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:40,843 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:45:41,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:42,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:45,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:45:50,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:45:52,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:45:52,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:53,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:53,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:45:53,755 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:45:53,755 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:45:53,756 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:45:53,856 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:45:53,856 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:45:54,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:45:56,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:45:59,662 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:46:02,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:02,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:46:02,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:03,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:04,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:46:04,499 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:46:04,499 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:46:04,516 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:46:04,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:04,880 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:46:05,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:05,894 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:46:06,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:07,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:07,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:12,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:17,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:19,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:46:19,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:20,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:21,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:46:21,107 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:46:21,107 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:46:21,109 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:46:21,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:21,919 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:46:21,920 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:46:23,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:28,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:29,665 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:46:33,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:38,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:42,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:46:42,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:43,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:43,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:44,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:46:44,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:46:44,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:46:44,096 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:46:44,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:46:44,954 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:46:45,971 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:46:46,971 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:46:49,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:54,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:59,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:46:59,664 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:47:03,467 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:47:03,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:04,467 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:04,511 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:47:05,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:47:05,115 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:47:05,115 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:47:05,116 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:47:05,467 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:06,003 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:47:06,003 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:47:06,467 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:07,468 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:08,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:10,467 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:47:15,467 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:47:20,468 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:47:24,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:47:24,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:25,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:25,735 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:47:26,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:47:26,667 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:47:26,667 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:47:26,669 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:47:26,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:27,048 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:47:27,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:28,049 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:47:28,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:29,666 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:47:29,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:30,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:31,692 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:47:31,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:32,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:33,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:34,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:35,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:36,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:37,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:47:42,693 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:47:45,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:47:45,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:46,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:47,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:47:47,067 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:47:47,067 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:47:47,068 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:47:47,080 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:47:47,624 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:48,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:47:48,081 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:47:53,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:47:58,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:47:58,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:47:58,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:59,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:47:59,668 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:47:59,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:47:59,743 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:47:59,743 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:47:59,744 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:48:00,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:00,092 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:48:00,092 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:48:03,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:48:08,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:48:13,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:48:14,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:48:14,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:15,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:15,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:48:15,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:48:15,459 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:48:15,460 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:48:16,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:16,119 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:48:16,119 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:48:17,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:19,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:48:24,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:48:29,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:48:29,668 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:48:32,345 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:48:32,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:33,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:33,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:48:33,675 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:48:33,675 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:48:33,676 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:48:34,137 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:48:34,137 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:48:34,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:34,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:48:35,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:36,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:37,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:38,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:39,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:40,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:48:40,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:41,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:45,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:48:50,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:48:55,347 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:48:58,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:48:58,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:59,669 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:48:59,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:48:59,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:48:59,963 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:48:59,963 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:48:59,965 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:49:00,222 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:49:00,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:00,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:02,224 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:49:05,966 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:10,966 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:15,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:49:15,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:16,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:16,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:16,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:49:16,447 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:49:16,447 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:49:16,449 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:49:17,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:17,238 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:49:18,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:18,238 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:49:19,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:20,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:21,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:22,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:22,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:27,024 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:29,671 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:49:32,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:33,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:49:33,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:34,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:34,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:49:34,807 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:49:34,807 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:49:34,808 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:49:35,283 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:49:35,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:36,283 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:49:36,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:37,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:37,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:38,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:39,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:40,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:41,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:42,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:43,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:43,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:44,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:45,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:46,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:47,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:48,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:48,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:49,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:50,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:51,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:52,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:53,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:49:53,294 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:49:53,295 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:49:53,297 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:49:53,297 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:49:53,444 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:53,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:54,298 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:49:54,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:55,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:56,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:57,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:58,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:49:58,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:59,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:49:59,705 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:50:00,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:01,447 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:03,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:50:08,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:50:09,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:50:09,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:10,196 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:10,978 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:50:10,979 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:50:10,979 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:50:10,981 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:50:11,196 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:11,328 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:50:12,196 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:12,330 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:50:13,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:50:18,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:50:23,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:50:23,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:24,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:24,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:50:24,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:50:24,871 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:50:24,871 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:50:24,872 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:50:25,343 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:50:25,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:26,350 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:50:26,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:27,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:28,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:29,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:29,672 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:50:29,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:50:30,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:31,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:32,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:33,484 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:35,483 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:50:36,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:37,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:38,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:50:38,511 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:50:38,511 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:50:38,513 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:50:38,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:39,364 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:50:40,365 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:50:40,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:50:45,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:50:50,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:50:55,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:50:57,672 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:50:57,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:58,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:58,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:50:58,911 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:50:58,911 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:50:58,912 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:50:59,383 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:50:59,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:50:59,674 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:51:00,384 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:51:00,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:00,675 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:01,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:02,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:03,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:04,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:05,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:06,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:06,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:11,673 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:13,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:51:13,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:14,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:14,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:51:14,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:51:14,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:51:14,721 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:51:15,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:15,416 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:51:16,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:16,416 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:51:16,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:17,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:18,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:19,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:20,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:22,200 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:22,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:23,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:23,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:51:23,687 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:51:23,687 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:51:23,688 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:51:24,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:24,425 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:51:24,426 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:51:25,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:26,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:27,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:27,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:28,225 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:29,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:29,677 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:51:33,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:38,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:43,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:45,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:51:45,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:46,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:46,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:51:46,935 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:51:46,935 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:51:46,937 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:51:47,448 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:51:47,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:48,449 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:51:48,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:48,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:49,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:51:53,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:58,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:51:59,676 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:52:01,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:52:01,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:02,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:02,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:52:02,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:52:02,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:52:02,941 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:52:03,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:03,464 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:52:03,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:52:04,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:04,465 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:52:08,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:52:13,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:52:14,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:52:14,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:15,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:15,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:52:15,899 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:52:15,899 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:52:15,901 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:52:16,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:16,488 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:52:16,488 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:52:17,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:18,469 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:18,489 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:52:19,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:52:19,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:24,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:52:29,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:52:29,677 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:52:34,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:52:34,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:35,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:35,291 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:52:35,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:52:35,621 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:52:35,621 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:52:35,638 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:52:36,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:36,506 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:52:36,507 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:52:37,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:38,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:39,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:40,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:41,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:41,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:52:42,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:43,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:44,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:47,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:52:52,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:52:52,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:53,003 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:53,047 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:52:53,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:52:53,655 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:52:53,655 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:52:53,657 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:52:54,003 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:52:54,535 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:52:54,535 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:52:58,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:52:59,679 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:53:03,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:53:08,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:53:10,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:53:10,650 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:11,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:12,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:53:12,147 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:53:12,147 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:53:12,149 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:53:12,552 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:53:12,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:13,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:14,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:53:14,554 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:53:14,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:15,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:16,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:17,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:18,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:19,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:53:19,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:20,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:21,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:22,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:23,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:24,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:53:24,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:25,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:26,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:27,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:28,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:29,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:53:29,431 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:53:29,431 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:53:29,433 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:53:29,598 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:53:29,609 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:29,682 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:53:29,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:53:30,599 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:53:34,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:53:39,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:53:39,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:53:39,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:40,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:41,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:53:41,371 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:53:41,371 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:53:41,373 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:53:41,609 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:53:41,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:42,610 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:53:42,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:43,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:44,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:53:44,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:45,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:46,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:49,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:53:50,212 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:51,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:51,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:53:51,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:53:51,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:53:51,720 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:53:52,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:52,618 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:53:52,619 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:53:53,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:54,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:55,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:53:55,212 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:56,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:57,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:58,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:59,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:53:59,681 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:54:00,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:01,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:54:01,212 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:02,211 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:03,212 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:05,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:06,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:06,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:54:07,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:54:07,323 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:54:07,323 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:54:07,325 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:54:07,630 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:54:07,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:08,631 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:54:08,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:09,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:10,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:11,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:12,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:12,955 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:54:17,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:54:18,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:54:18,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:19,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:19,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:54:19,931 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:54:19,931 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:54:19,932 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:54:20,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:20,642 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:54:21,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:22,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:22,644 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:54:23,933 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:54:28,933 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:54:29,683 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:54:34,684 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:54:39,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:54:39,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:40,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:40,159 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:54:40,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:54:40,891 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:54:40,891 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:54:40,893 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:54:41,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:41,661 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:54:42,662 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:54:45,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:54:49,295 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:54:49,338 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:50,295 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:50,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:54:50,699 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:54:50,699 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:54:50,700 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:54:51,295 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:51,669 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:54:51,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:54:52,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:52,670 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:54:53,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:54,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:55,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:54:57,296 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:54:59,686 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:55:02,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:55:07,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:55:12,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:55:17,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:55:21,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:55:21,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:22,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:22,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:55:23,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:55:23,504 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:55:23,504 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:55:23,505 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:55:23,694 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:55:23,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:24,694 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:55:24,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:25,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:26,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:27,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:28,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:28,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:55:29,685 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:55:29,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:30,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:34,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:55:35,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:36,364 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:36,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:55:36,791 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:55:36,791 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:55:36,793 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:55:37,364 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:37,706 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:55:38,707 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:55:39,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:55:44,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:55:49,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:55:52,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:55:52,362 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:53,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:53,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:55:53,727 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:55:53,727 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:55:53,728 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:55:54,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:54,720 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:55:54,720 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:55:55,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:55:55,729 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:55:59,687 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:56:01,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:56:06,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:56:11,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:56:14,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:56:14,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:15,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:16,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:56:16,667 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:56:16,667 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:56:16,670 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:56:16,738 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:56:16,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:17,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:56:17,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:18,740 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:56:18,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:19,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:20,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:22,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:56:27,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:56:29,689 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:56:33,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:56:38,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:56:39,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:56:39,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:40,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:40,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:56:40,631 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:56:40,631 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:56:40,633 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:56:40,758 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:56:41,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:42,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:42,759 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:56:43,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:44,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:56:44,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:49,205 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:56:54,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:56:55,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:56:55,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:56,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:56,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:56:56,751 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:56:56,751 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:56:56,754 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:56:56,772 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:56:57,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:58,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:58,774 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:56:59,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:56:59,689 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:56:59,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:00,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:01,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:02,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:03,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:04,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:05,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:05,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:06,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:07,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:08,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:09,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:10,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:10,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:11,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:12,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:13,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:14,221 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:15,220 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:15,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:15,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:57:15,423 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:57:15,424 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:57:15,425 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:57:15,786 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:57:16,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:16,787 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:57:17,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:18,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:19,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:20,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:20,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:21,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:22,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:23,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:24,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:25,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:26,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:26,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:27,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:28,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:29,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:57:29,055 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:57:29,055 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:57:29,057 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:57:29,375 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:29,691 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:57:29,799 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:57:30,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:30,800 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:57:31,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:31,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:32,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:33,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:34,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:35,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:36,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:37,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:37,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:38,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:39,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:40,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:40,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:57:40,935 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:57:40,935 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:57:40,937 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:57:41,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:41,811 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:57:42,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:42,812 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:57:42,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:43,376 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:47,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:49,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:57:49,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:50,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:50,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:57:50,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:57:50,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:57:50,940 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:57:51,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:57:51,821 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:57:52,822 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:57:52,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:57,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:57:59,694 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:58:03,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:58:08,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:58:09,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:58:09,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:10,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:10,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:58:10,858 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:58:10,859 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:58:10,860 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:58:11,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:11,848 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:58:12,849 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:58:13,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:58:18,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:58:23,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:58:28,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:58:29,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:58:29,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:29,693 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:58:30,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:30,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:58:30,523 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:58:30,523 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:58:30,525 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:58:30,864 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:58:31,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:32,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:32,866 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:58:33,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:34,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:58:34,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:35,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:36,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:37,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:38,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:39,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:40,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:58:40,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:41,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:42,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:43,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:44,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:45,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:46,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:58:46,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:46,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:58:46,767 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:58:46,767 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:58:46,768 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:58:46,879 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:58:47,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:48,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:48,880 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:58:49,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:50,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:51,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:58:51,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:52,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:58:56,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:58:59,694 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:59:01,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:59:01,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:59:01,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:02,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:03,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:59:03,511 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:59:03,511 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:59:03,512 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:59:03,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:03,893 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:59:04,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:04,894 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:59:05,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:07,513 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:59:12,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:59:17,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:59:21,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:59:21,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:22,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:22,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:59:22,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:59:22,851 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:59:22,851 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:59:22,853 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:59:22,925 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:59:23,305 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:24,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:24,926 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:59:25,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:26,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:27,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:28,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:59:29,696 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 14:59:31,588 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:32,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:33,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:59:33,275 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:59:33,275 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:59:33,276 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:59:33,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:33,950 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:59:33,950 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:59:34,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:59:34,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:35,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:39,277 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:59:41,327 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 14:59:41,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:42,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:42,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 14:59:42,835 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 14:59:42,835 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 14:59:42,836 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 14:59:42,958 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 14:59:43,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:43,959 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 14:59:44,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:44,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:59:45,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:46,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:47,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:48,328 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 14:59:50,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:59:55,329 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 14:59:59,698 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:00:00,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:05,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:06,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:00:07,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:07,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:08,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:00:08,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:00:08,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:00:08,721 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:00:08,981 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:00:08,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:09,982 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:00:09,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:10,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:10,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:12,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:12,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:13,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:15,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:16,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:16,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:21,000 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:26,001 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:29,698 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:00:31,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:36,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:41,700 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:45,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:00:46,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:46,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:47,023 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:47,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:00:47,543 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:00:47,543 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:00:47,545 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:00:47,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:48,017 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:00:48,017 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:00:48,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:49,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:50,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:51,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:52,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:52,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:53,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:54,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:55,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:56,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:57,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:00:58,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:00:59,699 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:01:03,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:01:03,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:04,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:04,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:01:04,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:01:04,459 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:01:04,459 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:01:04,460 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:01:05,031 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:01:05,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:06,031 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:01:09,461 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:01:14,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:01:19,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:01:24,463 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:01:29,463 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:01:29,702 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:01:34,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:01:36,076 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:01:36,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:37,076 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:37,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:01:37,420 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:01:37,420 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:01:37,421 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:01:38,059 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:01:38,059 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:01:38,076 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:39,060 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:01:39,076 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:40,076 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:40,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:01:41,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:42,076 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:43,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:44,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:45,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:46,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:01:51,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:01:56,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:01:56,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:01:56,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:57,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:57,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:01:57,807 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:01:57,807 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:01:57,809 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:01:58,075 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:01:58,076 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:01:58,341 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:59,342 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:01:59,702 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:02:01,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:02:06,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:02:11,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:02:16,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:02:16,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:17,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:17,623 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:02:17,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:02:17,983 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:02:17,983 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:02:17,984 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:02:18,095 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:02:18,095 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:02:18,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:19,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:20,097 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:02:20,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:21,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:22,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:23,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:23,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:02:24,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:25,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:26,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:27,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:28,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:29,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:02:29,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:29,703 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:02:29,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:02:29,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:02:29,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:02:29,796 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:02:30,106 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:02:30,106 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:02:30,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:31,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:32,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:33,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:34,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:02:34,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:35,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:36,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:37,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:40,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:02:43,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:02:43,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:44,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:45,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:02:45,271 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:02:45,271 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:02:45,273 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:02:45,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:46,120 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:02:46,120 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:02:46,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:02:46,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:47,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:48,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:49,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:50,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:51,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:51,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:02:52,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:53,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:54,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:55,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:56,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:57,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:57,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:02:58,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:02:59,707 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:02:59,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:02:59,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:02:59,747 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:02:59,749 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:02:59,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:00,138 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:03:00,139 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:03:00,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:01,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:02,864 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:03,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:03:03,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:08,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:03:12,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:03:12,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:13,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:14,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:03:14,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:03:14,107 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:03:14,107 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:03:14,108 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:03:14,151 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:03:14,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:15,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:16,153 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:03:16,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:17,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:18,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:19,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:19,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:03:20,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:21,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:22,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:23,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:24,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:24,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:03:25,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:26,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:27,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:28,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:29,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:29,706 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:03:29,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:03:30,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:31,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:32,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:33,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:34,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:35,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:03:35,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:36,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:36,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:03:36,923 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:03:36,923 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:03:36,925 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:03:37,185 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:03:37,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:03:38,186 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:03:40,679 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:03:45,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:03:50,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:03:55,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:03:59,707 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:04:00,708 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:05,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:10,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:13,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:04:13,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:14,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:15,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:04:15,511 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:04:15,511 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:04:15,514 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:04:15,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:16,218 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:04:16,218 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:04:16,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:16,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:17,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:18,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:19,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:20,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:21,945 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:24,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:25,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:26,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:04:26,047 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:04:26,047 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:04:26,049 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:04:26,228 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:04:26,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:27,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:27,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:28,229 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:04:29,709 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:04:32,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:37,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:42,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:43,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:04:43,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:44,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:45,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:04:45,199 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:04:45,199 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:04:45,200 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:04:45,244 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:04:45,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:46,245 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:04:46,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:47,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:48,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:48,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:49,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:50,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:04:53,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:58,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:04:59,712 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:05:03,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:05:08,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:05:12,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:05:12,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:13,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:13,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:05:13,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:05:13,947 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:05:13,947 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:05:13,949 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:05:14,240 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:14,266 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:05:14,266 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:05:15,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:16,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:17,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:18,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:19,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:19,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:05:20,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:21,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:24,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:05:29,243 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:05:29,711 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:05:33,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:05:33,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:34,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:34,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:05:34,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:05:34,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:05:34,747 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:05:34,748 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:05:35,284 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:05:35,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:36,285 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:05:36,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:37,309 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:38,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:39,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:40,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:05:40,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:45,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:05:50,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:05:55,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:05:56,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:05:56,922 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:57,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:58,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:05:58,335 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:05:58,335 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:05:58,336 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:05:58,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:05:59,303 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:05:59,713 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:06:00,304 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:06:00,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:05,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:10,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:15,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:16,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:06:16,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:17,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:18,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:06:18,283 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:06:18,283 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:06:18,284 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:06:18,318 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:06:18,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:19,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:20,320 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:06:20,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:21,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:21,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:22,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:23,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:26,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:29,729 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:06:31,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:06:31,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:32,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:32,059 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:32,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:06:32,643 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:06:32,643 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:06:32,645 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:06:33,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:33,332 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:06:34,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:34,332 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:06:35,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:36,016 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:37,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:38,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:38,016 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:39,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:40,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:41,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:42,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:43,016 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:44,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:44,016 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:06:49,016 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:54,016 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:59,017 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:06:59,715 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:07:03,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:07:03,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:04,686 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:04,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:07:05,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:07:05,411 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:07:05,411 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:07:05,413 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:07:05,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:06,371 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:07:06,371 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:07:06,687 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:10,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:07:15,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:07:20,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:07:22,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:07:22,674 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:23,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:24,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:07:24,123 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:07:24,123 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:07:24,125 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:07:24,398 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:07:24,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:25,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:26,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:07:26,400 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:07:26,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:27,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:28,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:29,633 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:29,717 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:07:30,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:31,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:07:36,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:07:36,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:37,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:37,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:07:37,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:07:37,583 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:07:37,583 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:07:37,584 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:07:38,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:38,412 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:07:38,412 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:07:39,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:42,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:07:47,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:07:50,175 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:07:50,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:51,176 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:51,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:07:51,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:07:51,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:07:51,929 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:07:52,176 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:52,425 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:07:52,425 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:07:52,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:07:53,176 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:54,176 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:55,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:56,176 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:07:58,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:07:59,718 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:08:03,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:08:08,720 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:08:08,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:08:08,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:09,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:10,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:08:10,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:08:10,399 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:08:10,401 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:08:10,441 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:08:10,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:12,443 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:08:14,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:08:19,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:08:24,403 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:08:27,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:08:27,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:28,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:29,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:08:29,371 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:08:29,371 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:08:29,373 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:08:29,458 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:08:29,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:29,720 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:08:29,721 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:08:30,459 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:08:30,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:31,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:32,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:33,591 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:35,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:08:40,592 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:08:45,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:08:50,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:08:53,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:08:53,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:54,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:55,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:08:55,103 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:08:55,103 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:08:55,106 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:08:55,480 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:08:55,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:56,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:08:56,481 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:08:56,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:08:59,721 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:09:01,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:06,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:09,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:09:09,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:10,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:10,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:09:10,739 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:09:10,739 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:09:10,756 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:09:11,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:11,495 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:09:11,757 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:12,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:12,496 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:09:13,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:14,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:17,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:22,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:27,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:29,722 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:09:32,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:37,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:39,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:09:39,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:40,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:40,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:09:40,551 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:09:40,551 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:09:40,553 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:09:40,579 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:09:41,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:42,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:42,581 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:09:43,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:43,553 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:44,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:45,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:49,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:54,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:56,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:09:56,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:57,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:58,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:09:58,715 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:09:58,715 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:09:58,717 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:09:58,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:09:59,610 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:09:59,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:09:59,726 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:09:59,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:00,611 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:10:00,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:01,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:02,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:03,809 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:04,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:10:08,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:09,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:09,963 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:10:10,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:10:10,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:10:10,003 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:10:10,020 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:10:10,621 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:10:10,621 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:10:10,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:11,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:12,623 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:10:12,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:13,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:14,656 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:15,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:10:15,658 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:16,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:17,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:18,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:19,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:20,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:21,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:21,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:10:22,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:23,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:24,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:25,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:26,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:27,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:10:27,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:28,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:10:28,563 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:10:28,563 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:10:28,564 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:10:28,637 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:10:28,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:29,725 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:10:30,639 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:10:32,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:10:37,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:10:42,727 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:10:43,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:10:43,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:44,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:44,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:10:44,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:10:44,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:10:44,720 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:10:45,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:45,652 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:10:46,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:46,652 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:10:47,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:48,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:10:48,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:49,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:50,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:10:53,317 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:10:58,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:10:59,726 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:11:00,009 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:11:00,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:01,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:01,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:11:01,563 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:11:01,563 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:11:01,564 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:11:01,664 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:11:02,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:02,665 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:11:03,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:03,565 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:11:04,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:05,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:06,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:07,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:08,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:09,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:11:14,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:11:19,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:11:24,012 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:11:27,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:11:27,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:28,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:29,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:11:29,262 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:11:29,263 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:11:29,263 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:11:29,264 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:11:29,690 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:11:29,746 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:11:29,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:30,691 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:11:30,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:31,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:34,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:11:39,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:11:44,748 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:11:49,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:11:54,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:11:54,282 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:55,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:55,283 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:11:55,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:11:55,683 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:11:55,683 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:11:55,685 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:11:55,712 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:11:56,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:56,714 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:11:57,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:58,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:11:59,729 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:12:00,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:05,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:10,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:12,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:12:12,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:13,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:13,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:12:13,931 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:12:13,931 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:12:13,933 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:12:14,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:14,733 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:12:14,733 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:12:15,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:15,933 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:16,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:17,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:18,504 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:19,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:21,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:25,973 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:12:26,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:26,973 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:27,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:27,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:12:27,727 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:12:27,727 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:12:27,729 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:12:27,745 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:12:27,973 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:28,746 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:12:28,973 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:29,730 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:12:32,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:37,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:40,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:12:40,406 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:41,364 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:41,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:12:41,771 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:12:41,771 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:12:41,773 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:12:42,365 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:12:42,758 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:12:42,758 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:12:42,773 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:47,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:52,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:57,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:12:59,734 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:13:00,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:13:00,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:01,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:01,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:13:01,891 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:13:01,891 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:13:01,893 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:13:02,452 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:02,775 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:13:02,776 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:13:02,893 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:03,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:04,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:05,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:06,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:08,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:13,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:18,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:23,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:28,455 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:29,732 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:13:33,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:38,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:43,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:44,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:13:44,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:45,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:45,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:13:45,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:13:45,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:13:45,929 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:13:46,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:46,824 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:13:46,825 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:13:47,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:48,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:48,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:49,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:50,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:51,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:52,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:53,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:54,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:54,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:55,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:55,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:13:55,839 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:13:55,839 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:13:55,841 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:13:56,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:56,833 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:13:56,833 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:13:57,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:58,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:59,222 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:13:59,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:13:59,734 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:14:00,223 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:04,735 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:14:09,736 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:14:14,736 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:14:15,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:14:15,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:16,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:17,578 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:14:17,579 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:14:17,579 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:14:17,581 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:14:17,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:17,880 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:14:18,881 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:14:20,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:14:25,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:14:26,003 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:14:26,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:27,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:27,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:14:27,463 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:14:27,463 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:14:27,465 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:14:27,890 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:14:28,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:28,891 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:14:29,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:29,735 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:14:30,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:30,736 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:14:31,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:32,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:33,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:34,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:35,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:36,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:14:36,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:37,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:38,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:39,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:40,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:41,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:14:41,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:42,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:43,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:46,004 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:14:47,431 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:48,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:48,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:14:48,827 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:14:48,827 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:14:48,829 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:14:48,942 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:14:49,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:50,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:14:50,944 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:14:51,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:14:56,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:14:59,744 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:15:01,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:15:01,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:02,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:02,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:15:03,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:15:03,107 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:15:03,107 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:15:03,109 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:15:03,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:03,970 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:15:04,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:04,970 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:15:05,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:06,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:07,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:08,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:08,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:15:09,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:10,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:11,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:12,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:13,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:14,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:14,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:15:15,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:16,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:17,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:18,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:18,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:15:18,851 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:15:18,851 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:15:18,853 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:15:18,998 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:15:19,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:19,853 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:15:20,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:21,000 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:15:21,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:22,688 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:23,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:24,689 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:25,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:15:29,738 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:15:30,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:15:30,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:31,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:31,135 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:15:31,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:15:31,547 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:15:31,547 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:15:31,564 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:15:32,010 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:15:32,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:33,011 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:15:33,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:34,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:35,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:36,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:37,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:37,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:15:38,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:39,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:42,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:15:42,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:43,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:44,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:15:44,103 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:15:44,103 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:15:44,105 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:15:44,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:45,028 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:15:45,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:46,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:15:47,029 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:15:47,105 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:15:52,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:15:57,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:15:59,739 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:16:02,740 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:16:04,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:16:04,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:05,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:05,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:16:05,547 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:16:05,547 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:16:05,549 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:16:06,058 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:16:06,059 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:16:06,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:08,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:16:13,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:16:18,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:16:21,555 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:16:21,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:22,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:22,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:16:22,959 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:16:22,959 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:16:22,960 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:16:23,089 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:16:23,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:23,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:16:24,090 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:16:24,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:25,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:26,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:27,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:29,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:16:29,776 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:16:34,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:16:39,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:16:40,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:16:40,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:41,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:42,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:16:42,379 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:16:42,379 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:16:42,380 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:16:42,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:43,108 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:16:44,109 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:16:45,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:16:50,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:16:53,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:16:53,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:54,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:54,966 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:16:54,967 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:16:54,967 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:16:54,969 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:16:55,119 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:16:55,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:55,969 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:16:56,135 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:16:56,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:16:59,742 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:17:01,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:06,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:11,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:16,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:20,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:17:20,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:21,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:21,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:22,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:17:22,423 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:17:22,423 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:17:22,425 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:17:22,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:23,167 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:17:23,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:24,167 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:17:27,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:29,744 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:17:32,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:37,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:39,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:17:39,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:40,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:41,263 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:17:41,264 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:17:41,264 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:17:41,266 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:17:41,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:42,184 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:17:42,184 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:17:42,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:43,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:43,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:44,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:45,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:48,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:53,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:57,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:17:57,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:58,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:58,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:17:58,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:17:58,951 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:17:58,951 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:17:58,952 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:17:59,215 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:17:59,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:17:59,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:18:00,216 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:18:00,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:01,480 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:02,481 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:03,481 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:04,481 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:18:04,481 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:05,481 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:09,481 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:18:12,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:18:12,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:13,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:14,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:18:14,111 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:18:14,111 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:18:14,113 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:18:14,228 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:18:14,716 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:15,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:18:15,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:16,243 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:18:16,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:17,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:18,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:19,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:20,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:20,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:18:21,717 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:25,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:18:29,746 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:18:30,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:18:33,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:18:33,334 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:34,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:35,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:18:35,187 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:18:35,187 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:18:35,203 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:18:35,274 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:18:35,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:36,204 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:18:36,275 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:18:36,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:37,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:38,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:39,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:40,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:41,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:41,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:18:42,292 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:43,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:44,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:45,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:46,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:46,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:18:47,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:48,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:49,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:50,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:51,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:52,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:52,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:18:53,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:54,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:55,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:56,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:57,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:58,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:18:58,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:59,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:18:59,747 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:19:00,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:00,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:19:00,359 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:19:00,359 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:19:00,361 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:19:01,295 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:19:01,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:02,296 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:19:02,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:03,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:19:03,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:04,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:05,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:06,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:07,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:08,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:09,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:19:09,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:10,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:11,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:12,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:13,311 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:14,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:15,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:19:15,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:16,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:17,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:18,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:19:18,291 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:19:18,291 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:19:18,293 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:19:18,312 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:18,326 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:19:20,328 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:19:21,293 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:19:26,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:19:29,748 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:19:31,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:19:36,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:19:36,844 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:19:36,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:37,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:38,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:19:38,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:19:38,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:19:38,196 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:19:38,374 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:19:38,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:39,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:19:40,375 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:19:42,197 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:19:47,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:19:52,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:19:57,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:19:59,751 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:20:02,752 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:20:07,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:20:12,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:20:15,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:20:15,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:16,140 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:16,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:20:16,871 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:20:16,871 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:20:16,872 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:20:17,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:17,419 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:20:17,873 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:20:18,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:18,420 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:20:19,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:20,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:21,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:22,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:23,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:23,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:20:24,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:25,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:26,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:27,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:28,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:29,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:29,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:20:29,751 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:20:30,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:31,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:32,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:33,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:34,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:35,141 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:20:35,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:36,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:37,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:40,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:20:40,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:41,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:42,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:20:42,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:20:42,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:20:42,112 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:20:42,440 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:20:42,440 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:20:42,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:43,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:44,442 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:20:44,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:45,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:20:45,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:46,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:47,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:48,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:49,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:50,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:20:50,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:51,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:52,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:52,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:20:52,975 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:20:52,975 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:20:52,977 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:20:53,450 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:20:53,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:54,450 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:20:54,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:55,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:20:55,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:56,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:57,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:58,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:59,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:20:59,753 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:21:00,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:01,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:01,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:02,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:03,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:04,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:05,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:06,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:06,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:07,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:08,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:09,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:11,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:16,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:21,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:25,227 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:21:25,270 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:26,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:26,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:26,969 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:21:26,969 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:21:26,970 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:21:26,971 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:21:27,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:27,474 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:21:28,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:28,475 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:21:29,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:29,757 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:21:30,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:31,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:32,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:32,228 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:37,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:42,229 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:45,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:21:45,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:46,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:47,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:21:47,083 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:21:47,083 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:21:47,085 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:21:47,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:47,493 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:21:48,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:48,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:48,494 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:21:49,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:50,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:51,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:52,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:53,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:53,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:54,267 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:21:58,268 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:21:59,755 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:22:03,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:22:03,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:04,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:04,191 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:22:04,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:22:04,527 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:22:04,527 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:22:04,528 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:22:05,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:05,528 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:22:06,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:06,528 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:22:07,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:08,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:09,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:10,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:22:15,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:22:20,151 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:22:25,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:22:29,757 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:22:30,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:22:30,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:22:30,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:31,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:32,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:22:32,287 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:22:32,287 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:22:32,289 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:22:32,577 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:22:32,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:33,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:34,578 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:22:34,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:35,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:22:35,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:36,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:37,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:38,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:39,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:40,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:22:41,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:22:46,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:22:51,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:22:56,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:22:59,774 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:23:01,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:06,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:09,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:23:09,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:10,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:11,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:23:11,563 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:23:11,563 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:23:11,565 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:23:11,648 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:23:11,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:12,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:12,649 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:23:12,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:13,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:14,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:15,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:17,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:22,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:27,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:29,760 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:23:33,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:38,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:39,351 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:23:39,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:40,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:40,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:23:40,775 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:23:40,775 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:23:40,777 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:23:41,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:41,705 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:23:42,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:42,706 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:23:43,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:43,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:44,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:45,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:46,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:47,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:48,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:49,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:54,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:57,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:23:57,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:58,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:23:59,355 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:23:59,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:23:59,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:23:59,399 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:23:59,416 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:23:59,739 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:23:59,761 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:23:59,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:00,740 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:24:00,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:01,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:04,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:24:07,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:24:07,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:08,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:09,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:24:09,027 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:24:09,027 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:24:09,033 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:24:09,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:09,748 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:24:10,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:24:10,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:10,749 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:24:11,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:15,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:24:20,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:24:25,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:24:29,762 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:24:30,763 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:24:35,764 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:24:38,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:24:38,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:39,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:39,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:24:39,903 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:24:39,903 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:24:39,905 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:24:40,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:40,800 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:24:40,800 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:24:40,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:24:41,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:42,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:43,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:44,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:45,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:46,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:46,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:24:47,359 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:48,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:49,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:50,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:51,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:52,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:52,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:24:53,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:54,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:55,360 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:56,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:56,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:24:56,427 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:24:56,427 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:24:56,429 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:24:56,828 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:24:57,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:57,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:24:58,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:58,830 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:24:59,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:24:59,766 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:25:00,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:02,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:07,768 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:12,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:15,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:25:15,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:16,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:17,149 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:25:17,150 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:25:17,150 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:25:17,152 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:25:17,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:17,845 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:25:18,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:18,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:18,846 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:25:23,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:28,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:29,765 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:25:33,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:36,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:25:36,782 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:37,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:38,142 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:25:38,143 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:25:38,143 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:25:38,145 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:25:38,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:38,864 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:25:39,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:39,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:40,741 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:40,866 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:25:44,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:49,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:51,188 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:25:51,230 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:52,188 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:52,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:25:52,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:25:52,695 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:25:52,696 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:25:52,909 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:25:53,188 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:54,188 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:25:54,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:54,911 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:25:59,697 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:25:59,766 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:26:04,767 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:26:06,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:26:06,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:07,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:07,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:26:07,823 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:26:07,823 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:26:07,824 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:26:07,922 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:26:08,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:08,923 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:26:09,332 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:09,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:26:14,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:26:19,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:26:20,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:26:21,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:21,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:22,354 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:26:22,355 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:26:22,355 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:26:22,356 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:26:22,950 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:26:22,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:24,951 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:26:25,357 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:26:29,769 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:26:30,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:26:33,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:26:33,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:34,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:35,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:26:35,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:26:35,175 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:26:35,177 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:26:35,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:35,963 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:26:36,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:26:36,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:36,964 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:26:37,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:38,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:39,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:41,407 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:26:46,408 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:26:47,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:26:47,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:48,036 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:48,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:26:48,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:26:48,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:26:48,813 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:26:48,975 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:26:49,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:50,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:50,977 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:26:51,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:51,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:26:52,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:26:57,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:26:59,769 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:26:59,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:26:59,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:00,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:01,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:27:01,571 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:27:01,571 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:27:01,573 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:27:01,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:01,988 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:27:02,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:27:02,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:02,989 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:27:03,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:04,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:05,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:06,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:07,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:27:07,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:12,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:27:15,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:27:15,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:16,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:17,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:27:17,163 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:27:17,163 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:27:17,165 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:27:17,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:18,001 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:27:18,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:27:18,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:19,002 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:27:19,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:20,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:21,753 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:22,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:23,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:23,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:27:24,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:28,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:27:29,770 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:27:33,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:27:38,771 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:27:43,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:27:43,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:44,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:44,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:27:44,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:27:44,959 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:27:44,959 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:27:44,960 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:27:45,023 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:27:45,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:46,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:47,025 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:27:47,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:48,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:27:50,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:27:55,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:27:59,785 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:28:00,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:05,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:10,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:15,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:28:15,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:28:16,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:28:16,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:16,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:28:16,903 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:28:16,903 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:28:16,904 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:28:17,051 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:28:17,571 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:28:19,053 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:28:21,905 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:26,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:29,773 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:28:32,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:28:32,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:28:32,055 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:33,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:28:33,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:28:33,439 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:28:33,439 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:28:33,441 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:28:34,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:28:34,081 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:28:35,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:28:35,082 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:28:36,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:28:37,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:28:38,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:28:38,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:39,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:28:43,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:48,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:53,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:58,016 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:28:59,774 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:29:03,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:29:08,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:29:08,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:29:09,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:09,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:10,390 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:29:10,391 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:29:10,391 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:29:10,392 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:29:10,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:11,116 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:29:11,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:12,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:13,117 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:29:13,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:29:13,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:14,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:18,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:29:19,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:20,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:21,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:29:21,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:29:21,235 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:29:21,236 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:29:21,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:22,156 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:29:22,157 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:29:22,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:23,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:24,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:29:24,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:29,775 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:29:29,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:29:30,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:29:30,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:31,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:31,678 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:29:31,679 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:29:31,679 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:29:31,680 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:29:32,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:32,166 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:29:32,166 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:29:33,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:34,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:29:35,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:29:40,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:29:45,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:29:50,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:29:55,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:29:59,794 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:30:00,795 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:00,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:30:01,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:01,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:02,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:30:02,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:30:02,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:30:02,368 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:30:02,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:03,200 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:30:03,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:04,201 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:30:04,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:05,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:05,981 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:10,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:15,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:19,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:30:19,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:20,447 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:20,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:30:20,851 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:30:20,851 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:30:20,852 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:30:21,215 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:30:21,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:21,853 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:22,216 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:30:26,853 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:29,778 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:30:32,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:37,779 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:42,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:47,780 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:49,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:30:49,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:50,498 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:50,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:30:50,959 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:30:50,959 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:30:50,960 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:30:51,239 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:30:51,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:52,240 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:30:52,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:52,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:53,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:54,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:55,499 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:56,503 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:30:58,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:30:59,779 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:31:01,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:31:01,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:02,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:03,062 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:31:03,063 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:31:03,063 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:31:03,064 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:31:03,249 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:31:03,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:04,065 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:31:04,250 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:31:04,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:05,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:06,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:09,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:31:14,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:31:19,640 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:31:20,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:31:20,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:21,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:21,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:31:21,759 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:31:21,759 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:31:21,775 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:31:22,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:22,267 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:31:22,267 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:31:23,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:24,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:31:29,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:31:29,782 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:31:34,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:31:39,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:31:44,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:31:49,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:31:50,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:31:50,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:51,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:52,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:31:52,152 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:31:52,152 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:31:52,154 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:31:52,293 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:31:52,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:53,294 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:31:53,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:54,295 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:31:54,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:55,155 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:31:55,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:56,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:57,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:31:59,782 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:32:00,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:05,783 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:10,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:15,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:16,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:32:16,478 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:32:17,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:32:17,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:32:17,803 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:32:17,803 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:32:17,804 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:32:18,326 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:32:18,326 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:32:18,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:32:20,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:25,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:29,783 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:32:31,784 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:36,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:37,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:32:37,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:32:38,593 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:32:39,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:32:39,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:32:39,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:32:39,096 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:32:39,355 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:32:39,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:32:40,356 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:32:40,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:32:41,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:32:42,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:42,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:32:43,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:32:47,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:52,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:57,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:32:59,784 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:33:02,785 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:33:07,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:33:08,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:33:08,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:09,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:10,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:33:10,095 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:33:10,095 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:33:10,096 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:33:10,379 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:33:10,380 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:33:10,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:11,388 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:13,097 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:33:18,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:33:23,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:33:28,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:33:28,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:33:28,638 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:29,596 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:29,785 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:33:29,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:33:29,859 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:33:29,859 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:33:29,860 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:33:30,399 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:33:30,399 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:33:30,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:31,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:32,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:33,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:33,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:33:34,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:38,599 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:33:40,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:33:40,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:41,527 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:41,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:33:41,935 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:33:41,935 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:33:41,937 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:33:42,410 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:33:42,411 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:33:42,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:43,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:43,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:33:44,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:45,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:46,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:47,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:49,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:33:53,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:33:53,966 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:54,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:54,967 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:33:55,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:33:55,247 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:33:55,247 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:33:55,248 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:33:55,437 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:33:55,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:56,438 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:33:56,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:57,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:58,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:33:59,787 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:33:59,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:00,788 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:00,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:01,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:02,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:03,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:04,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:05,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:05,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:06,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:07,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:10,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:13,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:34:13,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:14,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:15,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:34:15,035 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:34:15,035 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:34:15,037 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:34:15,472 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:34:15,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:16,037 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:16,473 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:34:16,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:17,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:18,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:19,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:20,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:21,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:21,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:22,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:26,643 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:28,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:34:28,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:29,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:29,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:34:29,735 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:34:29,735 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:34:29,736 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:34:29,788 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:34:30,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:30,499 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:34:30,500 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:34:31,333 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:31,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:36,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:41,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:46,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:48,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:34:48,974 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:49,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:50,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:34:50,519 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:34:50,519 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:34:50,536 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:34:50,931 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:34:51,517 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:34:52,518 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:34:52,537 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:57,537 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:34:59,791 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:35:02,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:35:07,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:35:12,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:35:12,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:35:13,175 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:35:13,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:35:13,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:35:13,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:35:13,611 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:35:13,613 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:35:14,370 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:35:14,538 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:35:14,538 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:35:18,613 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:35:23,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:35:26,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:35:26,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:35:27,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:35:27,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:35:27,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:35:27,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:35:27,796 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:35:28,253 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:35:28,550 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:35:28,550 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:35:28,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:35:29,791 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:35:34,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:35:39,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:35:44,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:35:46,283 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:35:46,326 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:35:47,283 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:35:47,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:35:47,631 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:35:47,631 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:35:47,634 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:35:48,283 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:35:48,578 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:35:48,578 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:35:49,283 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:35:50,283 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:35:50,634 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:35:55,635 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:35:59,792 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:36:00,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:36:05,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:36:07,393 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:36:07,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:08,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:08,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:36:08,863 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:36:08,863 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:36:08,865 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:36:09,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:09,598 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:36:10,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:10,599 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:36:10,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:36:11,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:12,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:13,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:14,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:15,394 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:16,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:36:21,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:36:26,396 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:36:29,796 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:36:31,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:36:33,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:36:33,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:34,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:35,594 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:36:35,595 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:36:35,595 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:36:35,597 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:36:35,620 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:36:35,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:36,621 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:36:36,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:37,597 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:36:37,909 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:38,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:42,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:36:45,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:36:45,226 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:46,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:46,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:36:46,803 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:36:46,803 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:36:46,805 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:36:47,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:47,646 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:36:48,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:48,647 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:36:48,805 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:36:49,185 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:50,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:51,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:52,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:53,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:54,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:54,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:36:55,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:56,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:57,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:58,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:59,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:36:59,795 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:36:59,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:37:00,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:01,187 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:03,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:04,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:05,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:37:05,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:37:05,383 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:37:05,383 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:37:05,385 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:37:05,663 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:37:05,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:06,664 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:37:06,703 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:07,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:08,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:09,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:10,704 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:11,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:37:15,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:16,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:16,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:37:17,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:37:17,443 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:37:17,443 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:37:17,444 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:37:17,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:17,674 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:37:18,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:18,675 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:37:19,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:20,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:21,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:22,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:22,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:37:23,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:24,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:25,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:26,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:27,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:28,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:37:28,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:29,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:29,797 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:37:30,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:31,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:32,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:33,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:34,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:37:34,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:35,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:36,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:37,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:38,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:39,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:40,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:37:40,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:41,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:42,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:43,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:44,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:37:44,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:37:44,483 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:37:44,484 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:37:44,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:44,696 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:37:45,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:37:45,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:46,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:46,698 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:37:47,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:48,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:49,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:50,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:37:51,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:37:56,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:37:59,799 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:38:01,800 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:03,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:38:03,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:04,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:04,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:38:04,859 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:38:04,859 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:38:04,860 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:38:05,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:05,716 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:38:06,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:06,717 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:38:06,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:11,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:16,861 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:18,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:38:18,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:19,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:20,291 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:38:20,291 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:38:20,292 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:38:20,294 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:38:20,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:20,731 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:38:21,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:22,294 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:22,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:22,733 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:38:23,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:24,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:25,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:26,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:27,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:29,799 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:38:32,800 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:37,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:39,411 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:38:39,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:40,412 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:40,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:38:40,883 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:38:40,883 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:38:40,885 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:38:41,411 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:41,752 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:38:42,412 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:42,753 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:38:42,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:47,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:52,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:53,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:38:53,290 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:54,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:54,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:38:54,667 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:38:54,667 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:38:54,669 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:38:54,776 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:38:55,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:56,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:56,778 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:38:57,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:38:58,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:38:59,800 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:39:03,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:39:08,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:39:13,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:39:18,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:39:18,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:19,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:19,735 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:39:20,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:39:20,139 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:39:20,139 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:39:20,140 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:39:20,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:20,796 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:39:20,796 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:39:21,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:22,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:23,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:24,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:25,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:25,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:39:26,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:29,801 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:39:30,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:39:35,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:39:40,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:39:45,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:39:45,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:46,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:46,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:39:47,254 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:39:47,255 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:39:47,255 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:39:47,256 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:39:47,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:47,818 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:39:48,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:48,819 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:39:49,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:50,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:51,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:52,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:39:52,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:39:57,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:39:59,819 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:40:02,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:40:03,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:40:03,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:04,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:05,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:40:05,159 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:40:05,159 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:40:05,161 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:40:05,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:05,836 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:40:06,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:06,837 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:40:07,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:08,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:40:08,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:09,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:10,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:11,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:12,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:13,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:40:13,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:14,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:15,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:16,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:17,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:18,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:19,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:40:19,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:20,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:21,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:22,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:23,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:24,548 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:40:24,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:25,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:26,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:27,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:28,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:29,549 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:40:29,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:29,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:40:29,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:40:29,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:40:29,720 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:40:29,804 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:40:29,855 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:40:30,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:30,856 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:40:31,677 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:34,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:40:39,806 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:40:44,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:40:49,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:40:54,808 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:40:56,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:40:56,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:57,171 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:57,526 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:40:57,527 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:40:57,527 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:40:57,528 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:40:57,901 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:40:58,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:58,903 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:40:59,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:40:59,805 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:41:00,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:00,807 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:01,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:02,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:03,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:04,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:05,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:06,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:06,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:07,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:10,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:11,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:11,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:11,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:41:11,843 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:41:11,843 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:41:11,845 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:41:11,928 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:41:12,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:12,929 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:41:13,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:14,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:15,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:16,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:21,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:26,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:29,808 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:41:32,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:33,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:41:33,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:34,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:35,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:41:35,599 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:41:35,599 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:41:35,607 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:41:35,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:35,946 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:41:36,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:36,947 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:41:37,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:38,608 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:38,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:39,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:40,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:41,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:42,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:43,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:43,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:44,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:45,900 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:48,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:53,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:54,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:41:54,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:55,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:56,146 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:41:56,147 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:41:56,147 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:41:56,149 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:41:56,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:56,963 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:41:56,964 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:41:57,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:58,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:41:59,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:41:59,808 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:42:04,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:42:09,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:42:12,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:42:12,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:13,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:14,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:42:14,043 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:42:14,043 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:42:14,044 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:42:14,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:14,977 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:42:14,977 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:42:15,044 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:42:15,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:16,709 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:17,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:20,710 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:42:25,711 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:42:29,810 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:42:30,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:42:33,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:42:33,770 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:34,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:35,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:42:35,139 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:42:35,140 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:42:35,180 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:42:35,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:35,997 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:42:36,181 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:42:36,997 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:42:41,181 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:42:46,182 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:42:50,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:42:50,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:51,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:51,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:42:52,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:42:52,375 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:42:52,375 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:42:52,377 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:42:52,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:53,011 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:42:53,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:42:55,013 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:42:57,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:42:59,826 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:43:02,828 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:43:05,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:43:05,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:06,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:06,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:43:06,799 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:43:06,799 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:43:06,800 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:43:07,025 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:43:07,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:08,413 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:08,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:43:09,027 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:43:13,801 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:43:18,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:43:23,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:43:25,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:43:26,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:26,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:27,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:43:27,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:43:27,399 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:43:27,401 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:43:27,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:28,042 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:43:28,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:29,042 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:43:29,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:43:29,813 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:43:29,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:30,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:31,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:32,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:33,984 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:43:34,985 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:43:39,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:43:44,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:43:49,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:43:54,987 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:43:59,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:44:00,816 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:05,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:09,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:44:09,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:44:10,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:10,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:11,330 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:44:11,331 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:44:11,331 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:44:11,349 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:44:11,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:12,081 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:44:12,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:13,082 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:44:13,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:14,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:16,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:21,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:26,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:29,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:44:29,410 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:29,815 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:44:30,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:30,954 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:44:30,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:44:30,955 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:44:30,958 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:44:31,097 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:44:31,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:31,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:32,369 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:33,099 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:44:36,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:41,959 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:43,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:44:43,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:44,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:44,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:44:44,571 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:44:44,571 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:44:44,572 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:44:45,110 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:44:45,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:46,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:47,124 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:44:47,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:47,573 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:48,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:49,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:50,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:51,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:52,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:53,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:53,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:54,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:55,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:56,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:57,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:58,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:59,245 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:44:59,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:44:59,818 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:45:00,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:01,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:02,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:03,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:04,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:45:04,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:05,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:05,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:45:05,615 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:45:05,615 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:45:05,617 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:45:06,140 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:45:06,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:07,141 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:45:07,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:08,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:09,246 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:45:09,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:10,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:11,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:12,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:13,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:14,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:45:19,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:45:24,248 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:45:25,048 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:45:25,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:26,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:26,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:45:26,430 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:45:26,430 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:45:26,431 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:45:27,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:27,181 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:45:29,183 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:45:29,432 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:45:29,818 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:45:34,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:45:35,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:45:35,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:36,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:37,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:45:37,575 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:45:37,575 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:45:37,576 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:45:37,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:38,219 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:45:38,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:39,220 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:45:39,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:40,577 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:45:40,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:41,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:42,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:43,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:44,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:45,888 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:45:45,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:46,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:47,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:50,889 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:45:53,115 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:45:53,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:54,116 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:54,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:45:54,583 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:45:54,583 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:45:54,584 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:45:55,116 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:45:55,234 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:45:56,235 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:45:56,585 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:45:59,820 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:46:01,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:06,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:11,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:16,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:46:16,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:46:17,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:46:17,663 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:18,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:46:18,023 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:46:18,023 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:46:18,024 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:46:18,254 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:46:18,254 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:46:18,615 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:46:23,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:28,025 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:29,822 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:46:33,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:38,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:43,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:45,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:46:45,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:46:46,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:46:47,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:46:47,455 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:46:47,455 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:46:47,456 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:46:47,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:46:48,275 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:46:48,276 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:46:48,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:46:49,457 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:49,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:46:54,457 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:59,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:46:59,822 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:47:00,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:47:00,102 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:01,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:01,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:47:01,543 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:47:01,543 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:47:01,545 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:47:02,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:02,289 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:47:02,290 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:47:03,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:04,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:04,545 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:47:05,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:06,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:07,060 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:10,061 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:47:12,736 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:47:12,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:13,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:14,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:47:14,111 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:47:14,111 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:47:14,113 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:47:14,302 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:47:14,302 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:47:14,736 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:15,113 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:47:15,736 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:16,736 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:17,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:18,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:19,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:20,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:20,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:47:21,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:22,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:23,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:24,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:25,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:25,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:47:26,490 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:47:26,491 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:47:26,491 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:47:26,492 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:47:26,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:27,320 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:47:27,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:28,321 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:47:28,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:29,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:29,824 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:47:30,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:30,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:47:31,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:32,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:33,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:34,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:35,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:36,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:47:36,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:37,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:38,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:39,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:40,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:41,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:42,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:47:42,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:43,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:44,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:45,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:46,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:47,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:48,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:47:48,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:49,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:50,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:51,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:52,739 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:47:54,737 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:47:59,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:47:59,826 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:48:04,827 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:48:09,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:48:09,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:10,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:10,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:48:10,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:48:10,503 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:48:10,503 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:48:10,504 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:48:11,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:11,371 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:48:12,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:12,372 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:48:13,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:14,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:15,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:16,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:16,087 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:48:20,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:21,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:21,199 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:48:21,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:48:21,555 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:48:21,555 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:48:21,557 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:48:22,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:22,382 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:48:22,382 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:48:23,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:24,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:25,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:26,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:27,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:48:27,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:28,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:29,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:29,826 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:48:32,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:48:32,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:33,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:34,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:48:34,208 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:48:34,208 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:48:34,209 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:48:34,286 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:34,394 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:48:34,394 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:48:35,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:36,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:37,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:38,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:48:38,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:39,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:43,287 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:48:47,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:48:47,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:48,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:48,443 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:48:49,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:48:49,031 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:48:49,031 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:48:49,033 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:48:49,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:49,419 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:48:50,402 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:48:50,420 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:48:54,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:48:59,034 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:48:59,828 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:49:04,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:49:09,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:49:14,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:49:19,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:49:24,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:49:24,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:25,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:25,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:49:25,982 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:49:25,983 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:49:25,983 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:49:25,984 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:49:26,465 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:49:26,465 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:49:26,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:29,830 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:49:30,831 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:49:35,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:49:40,832 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:49:41,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:49:41,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:42,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:42,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:49:42,535 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:49:42,535 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:49:42,536 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:49:43,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:43,481 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:49:44,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:44,482 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:49:45,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:46,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:49:46,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:47,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:48,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:49,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:50,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:51,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:49:51,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:52,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:53,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:49:56,195 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:49:59,833 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:50:01,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:50:06,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:50:09,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:50:09,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:10,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:11,235 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:50:11,236 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:50:11,236 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:50:11,237 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:50:11,505 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:50:11,749 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:12,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:50:12,505 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:50:17,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:50:20,470 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:50:20,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:21,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:21,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:50:21,963 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:50:21,963 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:50:21,964 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:50:22,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:22,515 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:50:22,515 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:50:22,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:50:23,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:24,471 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:25,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:28,472 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:50:29,833 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:50:33,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:50:36,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:50:36,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:37,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:37,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:50:37,791 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:50:37,791 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:50:37,794 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:50:38,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:38,530 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:50:38,530 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:50:39,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:39,794 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:50:40,053 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:41,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:45,054 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:50:50,055 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:50:53,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:50:53,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:54,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:55,055 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:50:55,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:50:55,099 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:50:55,099 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:50:55,100 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:50:55,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:55,544 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:50:56,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:56,545 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:50:57,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:58,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:50:59,834 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:51:00,835 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:51:05,835 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:51:10,836 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:51:14,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:51:14,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:15,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:15,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:51:16,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:51:16,539 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:51:16,539 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:51:16,542 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:51:16,564 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:51:16,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:17,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:18,566 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:51:18,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:19,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:20,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:21,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:51:21,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:22,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:23,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:24,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:25,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:27,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:51:29,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:29,837 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:51:30,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:31,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:51:31,359 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:51:31,359 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:51:31,360 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:51:31,577 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:51:31,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:32,578 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:51:32,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:33,361 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:51:33,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:34,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:35,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:36,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:37,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:38,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:38,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:51:39,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:40,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:41,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:42,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:43,401 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:51:43,402 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:51:43,402 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:51:43,403 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:51:43,589 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:51:43,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:44,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:51:44,590 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:51:44,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:45,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:46,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:47,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:48,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:49,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:49,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:51:50,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:51,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:52,668 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:53,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:54,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:55,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:55,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:51:56,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:57,366 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:51:57,367 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:51:57,367 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:51:57,369 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:51:57,603 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:51:57,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:58,603 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:51:58,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:59,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:51:59,836 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:52:00,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:00,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:52:01,669 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:02,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:03,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:04,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:05,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:06,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:06,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:52:07,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:08,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:09,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:10,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:11,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:12,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:12,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:52:13,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:14,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:15,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:16,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:17,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:17,820 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:52:18,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:19,671 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:20,718 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:52:20,719 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:20,719 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:52:20,719 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:52:20,746 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:52:21,622 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:52:21,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:22,623 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:52:22,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:23,676 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:23,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:52:28,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:52:29,838 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:52:33,839 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:52:34,629 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:52:34,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:35,629 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:36,158 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:52:36,159 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:52:36,159 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:52:36,160 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:52:36,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:36,636 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:52:36,636 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:52:37,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:38,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:39,161 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:52:39,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:40,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:41,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:44,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:52:49,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:52:54,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:52:55,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:52:55,082 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:56,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:56,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:52:56,475 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:52:56,475 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:52:56,477 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:52:56,652 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:52:57,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:58,042 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:52:58,654 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:52:59,842 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:52:59,843 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:04,604 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:53:04,646 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:05,604 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:05,647 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:06,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:53:06,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:53:06,235 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:53:06,237 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:53:06,604 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:06,671 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:53:06,671 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:53:07,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:08,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:09,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:10,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:11,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:11,605 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:14,496 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:15,497 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:15,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:53:15,927 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:53:15,927 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:53:15,928 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:53:16,497 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:16,682 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:53:16,682 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:53:16,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:17,497 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:18,497 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:19,497 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:22,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:27,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:29,841 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:53:32,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:34,546 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:53:34,590 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:35,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:36,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:53:36,003 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:53:36,003 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:53:36,005 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:53:36,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:36,701 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:53:36,701 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:53:37,547 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:38,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:43,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:48,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:53,007 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:58,008 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:53:58,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:53:58,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:59,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:53:59,842 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:54:00,086 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:54:00,087 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:54:00,087 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:54:00,088 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:54:00,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:00,723 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:54:00,723 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:54:01,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:03,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:54:08,089 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:54:13,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:54:18,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:54:18,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:54:18,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:19,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:19,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:54:19,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:54:19,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:54:19,941 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:54:20,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:20,740 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:54:20,740 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:54:21,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:22,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:23,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:54:23,452 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:24,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:28,452 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:54:29,847 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:54:33,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:54:35,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:54:35,654 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:36,611 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:36,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:54:36,863 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:54:36,864 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:54:36,865 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:54:37,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:37,753 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:54:38,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:38,754 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:54:38,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:54:39,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:40,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:41,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:42,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:43,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:44,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:44,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:54:45,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:46,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:47,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:48,434 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:54:48,435 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:54:48,435 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:54:48,436 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:54:48,764 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:54:48,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:49,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:54:50,437 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:54:50,766 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:54:55,612 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:54:59,846 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:55:00,847 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:05,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:10,848 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:15,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:55:15,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:55:16,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:55:16,587 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:17,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:55:17,299 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:55:17,299 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:55:17,301 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:55:17,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:55:17,788 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:55:18,544 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:55:18,789 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:55:22,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:27,302 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:29,848 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:55:32,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:37,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:42,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:43,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:55:43,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:55:44,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:55:45,518 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:55:45,519 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:55:45,519 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:55:45,520 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:55:45,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:55:45,812 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:55:46,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:55:46,813 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:55:47,776 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:55:48,521 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:53,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:58,522 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:55:59,850 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:56:03,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:56:08,852 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:56:12,969 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:56:13,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:56:13,969 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:56:14,011 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:56:14,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:56:14,487 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:56:14,487 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:56:14,489 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:56:14,838 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:56:14,969 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:56:15,969 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:56:16,840 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:56:16,969 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:56:17,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:56:18,970 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:56:19,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:56:24,971 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:56:29,854 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:56:30,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:56:35,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:56:40,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:56:45,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:56:50,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:56:55,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:56:57,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:56:57,078 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:56:58,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:56:58,758 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:56:58,759 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:56:58,759 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:56:58,761 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:56:58,883 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:56:59,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:56:59,853 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:57:00,035 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:00,885 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:57:01,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:06,855 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:11,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:16,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:21,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:22,463 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:57:22,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:23,463 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:23,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:57:23,831 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:57:23,831 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:57:23,833 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:57:23,901 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:57:24,463 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:24,902 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:57:25,463 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:26,463 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:27,464 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:27,465 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:28,463 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:29,855 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:57:32,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:37,857 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:38,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:57:38,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:39,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:40,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:57:40,239 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:57:40,239 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:57:40,241 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:57:40,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:40,916 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:57:40,916 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:57:41,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:42,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:43,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:43,812 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:44,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:48,812 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:53,812 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:57,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:57:57,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:58,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:58,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:57:58,731 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:57:58,731 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:57:58,733 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:57:58,930 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:57:59,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:57:59,733 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:57:59,858 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:58:00,331 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:00,932 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:58:04,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:58:09,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:58:13,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:58:13,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:14,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:15,110 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:58:15,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:58:15,151 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:58:15,151 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:58:15,164 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:58:15,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:15,946 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:58:16,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:16,946 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:58:17,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:18,708 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:19,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:20,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:20,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:58:21,707 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:25,708 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:58:29,858 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:58:30,859 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:58:35,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:58:40,860 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:58:42,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 15:58:42,250 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:43,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:43,581 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:58:43,582 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:58:43,582 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:58:43,583 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:58:43,965 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:58:44,206 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:44,966 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:58:45,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:46,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:46,584 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:58:47,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:48,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:49,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:50,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:51,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:52,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:58:52,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:53,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:54,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:57,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:58,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:58,207 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:58:58,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:58:58,507 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:58:58,507 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:58:58,508 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:58:58,979 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:58:59,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:58:59,860 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:59:00,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:00,980 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:59:01,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:02,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:03,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:03,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:59:04,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:05,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:06,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:07,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:08,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:09,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:09,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:59:10,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:11,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:12,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:13,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:14,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:15,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:15,208 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:59:16,164 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:17,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:18,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:18,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 15:59:18,883 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 15:59:18,883 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 15:59:18,885 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 15:59:18,995 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 15:59:19,165 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 15:59:20,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:59:20,997 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 15:59:25,886 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:59:29,862 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 15:59:31,863 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:59:36,864 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:59:41,864 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:59:46,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:59:51,865 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:59:56,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 15:59:59,865 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:00:01,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:00:01,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:02,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:02,603 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:00:03,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:00:03,031 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:00:03,031 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:00:03,032 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:00:03,039 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:00:03,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:04,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:05,041 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:00:05,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:06,559 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:08,561 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:00:13,561 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:00:18,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:00:23,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:00:27,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:00:27,314 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:28,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:28,651 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:00:28,694 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:00:28,695 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:00:28,695 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:00:28,696 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:00:29,058 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:00:29,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:29,865 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:00:30,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:31,060 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:00:31,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:32,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:33,272 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:34,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:34,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:00:35,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:36,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:37,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:38,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:39,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:40,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:00:40,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:40,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:00:40,847 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:00:40,847 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:00:40,849 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:00:41,069 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:00:41,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:42,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:43,071 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:00:43,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:44,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:45,273 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:00:45,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:46,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:50,274 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:00:51,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:52,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:52,842 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:00:52,843 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:00:52,843 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:00:52,845 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:00:53,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:53,095 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:00:54,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:55,068 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:55,097 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:00:55,845 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:00:56,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:57,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:58,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:59,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:00:59,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:01:00,868 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:05,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:10,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:11,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:01:11,894 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:12,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:13,234 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:01:13,235 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:01:13,235 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:01:13,236 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:01:13,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:14,111 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:01:14,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:15,112 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:01:15,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:16,237 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:16,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:17,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:18,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:19,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:20,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:21,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:21,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:22,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:24,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:25,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:26,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:01:26,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:01:26,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:01:26,197 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:01:26,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:27,122 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:01:27,122 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:01:27,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:27,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:28,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:29,886 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:01:29,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:30,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:31,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:32,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:32,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:37,851 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:38,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:01:38,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:39,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:40,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:01:40,047 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:01:40,047 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:01:40,049 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:01:40,134 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:01:40,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:41,135 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:01:41,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:42,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:43,049 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:48,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:53,050 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:58,051 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:01:59,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:01:59,670 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:01:59,870 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:02:00,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:01,281 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:02:01,282 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:02:01,282 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:02:01,283 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:02:01,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:02,151 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:02:02,630 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:03,152 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:02:03,284 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:02:03,631 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:08,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:02:13,285 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:02:16,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:02:16,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:17,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:18,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:02:18,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:02:18,175 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:02:18,176 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:02:18,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:19,175 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:02:19,175 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:02:19,177 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:02:19,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:20,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:21,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:22,680 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:23,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:24,681 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:02:29,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:02:29,872 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:02:33,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:02:33,462 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:34,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:35,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:02:35,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:02:35,175 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:02:35,175 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:02:35,192 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:02:35,192 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:02:35,420 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:36,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:37,194 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:02:40,192 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:02:45,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:02:50,193 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:02:51,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:02:51,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:52,515 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:53,247 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:02:53,247 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:02:53,247 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:02:53,249 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:02:53,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:54,207 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:02:54,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:55,208 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:02:55,249 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:02:55,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:56,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:57,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:58,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:59,517 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:02:59,882 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:03:00,516 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:00,883 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:05,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:10,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:15,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:20,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:21,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:03:21,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:22,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:23,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:03:23,167 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:03:23,167 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:03:23,169 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:03:23,247 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:03:23,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:24,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:25,280 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:03:25,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:26,170 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:26,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:27,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:28,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:29,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:29,875 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:03:30,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:31,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:31,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:32,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:33,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:35,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:36,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:36,574 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:03:36,575 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:03:36,575 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:03:36,576 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:03:36,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:37,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:37,292 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:03:38,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:39,152 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:39,294 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:03:40,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:41,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:42,153 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:42,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:47,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:52,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:55,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:03:55,734 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:56,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:57,238 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:03:57,239 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:03:57,239 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:03:57,241 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:03:57,309 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:03:57,691 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:03:58,241 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:03:59,311 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:03:59,877 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:04:03,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:04:08,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:04:13,879 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:04:17,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:04:17,858 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:18,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:19,224 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:04:19,266 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:04:19,267 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:04:19,267 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:04:19,285 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:04:19,343 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:04:19,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:20,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:21,345 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:04:21,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:22,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:24,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:04:29,819 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:04:29,878 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:04:30,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:04:30,258 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:31,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:31,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:04:31,615 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:04:31,615 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:04:31,617 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:04:32,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:32,356 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:04:33,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:33,357 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:04:34,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:35,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:04:35,216 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:36,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:37,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:38,215 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:39,216 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:40,216 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:04:40,216 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:04:45,216 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:04:50,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:04:55,217 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:04:59,883 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:05:00,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:05,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:08,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:05:08,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:09,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:10,506 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:05:10,507 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:05:10,507 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:05:10,509 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:05:10,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:11,400 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:05:11,510 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:11,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:12,401 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:05:12,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:13,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:14,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:16,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:21,031 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:05:21,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:22,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:22,075 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:22,448 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:05:22,449 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:05:22,449 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:05:22,450 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:05:23,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:23,419 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:05:24,033 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:24,420 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:05:27,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:29,883 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:05:32,884 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:37,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:05:37,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:38,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:38,487 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:39,114 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:05:39,115 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:05:39,115 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:05:39,117 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:05:39,434 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:05:39,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:40,435 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:05:40,445 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:44,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:49,118 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:54,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:59,119 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:05:59,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:05:59,754 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:05:59,885 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:06:00,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:01,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:06:01,139 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:06:01,139 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:06:01,157 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:06:01,456 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:06:01,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:02,456 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:06:02,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:03,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:04,157 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:06:04,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:05,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:06,712 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:07,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:08,713 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:09,714 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:06:12,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:13,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:14,350 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:06:14,351 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:06:14,351 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:06:14,352 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:06:14,468 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:06:14,468 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:06:14,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:15,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:06:15,665 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:16,484 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:06:16,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:17,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:18,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:20,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:06:25,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:06:29,889 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:06:30,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:06:33,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:06:33,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:34,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:34,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:06:34,743 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:06:34,743 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:06:34,744 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:06:35,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:35,501 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:06:36,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:36,502 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:06:36,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:06:37,080 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:41,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:06:46,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:06:51,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:06:56,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:06:57,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:06:57,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:58,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:58,550 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:06:58,551 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:06:58,551 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:06:58,553 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:06:59,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:06:59,539 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:06:59,889 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:07:00,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:00,540 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:07:01,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:01,890 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:02,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:03,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:04,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:05,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:06,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:07,083 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:07,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:08,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:09,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:09,810 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:07:09,811 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:07:09,811 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:07:09,812 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:07:10,084 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:10,591 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:07:10,591 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:07:11,085 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:12,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:17,813 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:22,814 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:25,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:07:25,162 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:26,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:26,938 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:07:26,939 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:07:26,939 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:07:26,940 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:07:27,124 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:27,605 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:07:27,941 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:28,121 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:28,606 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:07:29,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:29,891 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:07:30,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:31,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:32,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:33,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:38,122 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:43,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:48,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:52,111 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:07:52,154 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:53,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:53,155 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:53,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:07:53,599 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:07:53,599 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:07:53,601 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:07:53,641 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:07:54,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:54,642 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:07:55,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:56,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:57,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:07:59,112 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:07:59,900 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:08:04,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:08:09,901 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:08:10,395 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:08:10,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:11,396 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:11,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:08:11,819 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:08:11,819 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:08:11,821 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:08:12,396 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:12,673 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:08:12,673 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:08:13,396 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:14,396 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:15,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:08:20,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:08:25,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:08:26,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:08:26,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:27,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:28,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:08:28,134 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:08:28,134 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:08:28,135 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:08:28,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:28,686 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:08:28,686 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:08:29,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:29,895 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:08:30,683 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:30,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:08:35,896 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:08:40,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:08:45,897 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:08:49,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:08:49,802 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:50,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:51,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:08:51,150 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:08:51,151 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:08:51,151 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:08:51,153 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:08:51,716 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:08:51,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:52,716 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:08:52,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:53,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:54,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:55,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:56,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:08:56,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:57,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:58,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:59,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:08:59,897 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:09:00,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:01,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:01,898 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:02,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:03,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:04,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:05,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:06,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:07,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:07,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:08,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:09,690 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:09:09,691 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:09:09,691 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:09:09,693 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:09:09,760 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:09:09,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:10,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:10,761 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:09:11,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:12,759 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:12,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:13,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:14,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:15,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:16,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:17,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:17,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:18,761 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:22,760 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:24,539 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:09:24,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:25,539 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:25,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:09:25,991 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:09:25,991 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:09:25,992 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:09:26,539 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:26,774 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:09:26,775 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:09:27,540 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:27,993 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:29,898 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:09:33,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:38,899 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:41,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:09:41,179 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:42,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:42,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:09:42,618 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:09:42,618 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:09:42,619 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:09:42,788 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:09:43,137 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:09:44,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:44,790 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:09:49,620 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:54,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:59,621 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:09:59,900 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:09:59,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:10:00,046 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:00,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:01,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:10:01,847 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:10:01,847 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:10:01,849 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:10:01,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:02,810 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:10:02,810 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:10:02,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:04,849 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:10:09,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:10:13,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:10:13,074 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:14,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:14,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:10:14,419 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:10:14,419 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:10:14,421 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:10:14,823 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:10:15,030 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:15,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:10:16,840 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:10:20,421 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:10:25,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:10:29,902 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:10:30,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:10:35,903 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:10:37,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:10:37,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:38,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:39,010 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:10:39,011 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:10:39,011 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:10:39,013 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:10:39,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:39,860 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:10:40,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:40,861 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:10:41,013 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:10:41,568 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:46,014 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:10:51,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:10:56,015 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:10:56,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:10:56,774 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:57,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:58,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:10:58,539 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:10:58,539 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:10:58,540 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:10:58,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:58,923 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:10:59,732 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:10:59,904 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:11:00,732 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:00,925 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:11:01,730 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:01,732 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:02,732 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:03,732 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:06,731 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:09,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:11:09,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:10,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:11,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:11:11,019 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:11:11,019 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:11:11,021 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:11:11,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:11,936 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:11:12,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:12,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:12,936 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:11:13,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:14,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:15,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:16,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:17,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:17,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:18,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:19,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:23,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:28,440 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:29,909 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:11:33,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:38,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:11:38,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:39,026 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:39,067 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:39,346 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:11:39,346 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:11:39,346 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:11:39,348 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:11:39,976 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:11:40,977 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:11:44,348 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:49,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:54,349 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:57,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:11:57,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:58,452 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:58,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:11:58,839 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:11:58,839 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:11:58,840 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:11:58,993 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:11:59,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:11:59,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:11:59,907 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:12:00,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:00,995 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:12:01,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:02,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:03,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:04,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:05,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:05,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:12:06,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:07,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:08,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:09,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:10,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:10,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:12:11,453 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:12,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:13,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:14,194 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:12:14,195 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:12:14,195 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:12:14,197 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:12:14,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:15,007 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:12:15,007 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:12:15,454 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:16,197 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:12:21,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:12:26,198 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:12:27,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:12:27,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:28,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:29,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:12:29,187 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:12:29,187 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:12:29,189 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:12:29,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:29,909 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:12:30,022 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:12:30,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:31,023 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:12:31,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:31,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:12:32,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:33,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:34,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:35,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:36,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:37,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:12:42,725 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:12:47,726 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:12:49,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:12:50,006 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:50,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:51,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:12:51,439 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:12:51,439 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:12:51,441 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:12:51,965 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:12:52,048 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:12:53,049 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:12:53,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:12:58,442 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:12:59,925 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:13:03,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:13:03,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:03,627 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:13:04,582 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:04,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:13:04,931 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:13:04,931 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:13:04,934 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:13:05,061 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:13:05,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:06,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:07,063 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:13:07,583 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:08,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:13:13,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:13:15,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:13:15,598 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:16,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:17,018 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:13:17,019 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:13:17,019 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:13:17,020 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:13:17,073 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:13:17,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:18,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:19,021 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:13:19,075 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:13:19,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:20,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:21,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:22,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:23,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:24,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:24,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:13:25,558 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:29,911 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:13:29,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:13:34,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:13:35,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:13:35,214 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:36,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:36,662 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:13:36,663 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:13:36,663 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:13:36,666 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:13:37,105 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:13:37,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:39,106 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:13:40,666 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:13:45,667 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:13:47,994 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:13:48,038 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:48,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:49,422 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:13:49,423 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:13:49,423 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:13:49,425 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:13:49,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:50,116 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:13:50,995 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:51,117 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:13:51,425 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:13:55,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:56,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:56,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:13:57,614 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:13:57,615 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:13:57,615 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:13:57,617 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:13:57,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:58,125 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:13:58,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:59,126 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:13:59,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:13:59,913 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:14:00,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:01,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:02,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:14:03,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:04,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:05,644 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:14:05,644 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:14:05,645 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:14:05,646 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:14:05,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:06,134 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:14:06,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:07,135 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:14:07,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:08,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:08,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:14:09,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:10,817 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:14,910 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:14:19,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:14:24,911 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:14:29,912 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:14:29,916 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:14:32,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:14:32,830 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:33,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:34,378 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:14:34,379 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:14:34,379 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:14:34,381 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:14:34,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:35,185 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:14:35,185 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:14:35,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:14:35,789 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:14:40,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:14:45,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:14:50,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:14:55,383 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:14:59,916 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:15:00,917 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:05,918 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:06,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:15:06,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:07,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:08,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:15:08,127 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:15:08,127 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:15:08,129 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:15:08,226 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:15:08,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:09,227 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:15:09,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:10,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:11,130 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:11,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:12,315 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:16,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:21,316 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:22,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:15:22,494 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:23,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:23,882 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:15:23,883 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:15:23,883 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:15:23,884 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:15:24,241 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:15:24,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:25,242 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:15:25,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:26,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:26,885 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:27,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:28,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:29,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:29,917 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:15:30,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:31,919 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:35,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:36,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:36,766 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:15:36,767 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:15:36,767 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:15:36,769 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:15:37,254 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:15:37,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:37,769 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:38,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:39,256 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:15:39,352 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:40,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:41,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:43,353 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:45,773 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:46,773 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:47,554 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:15:47,555 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:15:47,555 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:15:47,557 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:15:47,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:48,269 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:15:48,557 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:48,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:49,269 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:15:49,777 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:50,778 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:15:53,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:58,775 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:15:59,921 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:16:03,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:16:08,923 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:16:13,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:16:18,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:16:23,063 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:16:23,106 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:16:24,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:16:24,107 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:16:24,514 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:16:24,515 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:16:24,515 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:16:24,516 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:16:25,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:16:25,297 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:16:26,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:16:27,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:16:27,299 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:16:28,064 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:16:29,925 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:16:29,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:16:34,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:16:39,927 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:16:44,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:16:44,438 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:16:45,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:16:45,439 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:16:45,850 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:16:45,851 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:16:45,851 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:16:45,853 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:16:46,314 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:16:46,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:16:47,315 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:16:47,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:16:48,397 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:16:50,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:16:55,854 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:16:59,923 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:17:00,924 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:05,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:10,925 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:15,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:20,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:21,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:17:21,458 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:22,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:22,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:17:22,835 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:17:22,835 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:17:22,837 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:17:23,350 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:17:23,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:24,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:25,351 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:17:25,415 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:26,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:29,925 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:17:31,926 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:34,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:17:34,738 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:35,695 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:36,090 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:17:36,091 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:17:36,091 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:17:36,093 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:17:36,362 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:17:36,696 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:37,093 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:37,363 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:17:42,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:47,094 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:50,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:17:50,682 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:51,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:52,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:52,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:17:52,124 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:17:52,124 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:17:52,125 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:17:52,378 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:17:52,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:53,379 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:17:53,639 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:17:57,126 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:17:59,928 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:18:02,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:18:03,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:18:03,322 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:04,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:04,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:18:04,699 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:18:04,699 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:18:04,701 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:18:05,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:05,391 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:18:06,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:07,393 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:18:08,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:18:13,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:18:18,702 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:18:22,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:18:22,998 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:23,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:23,999 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:18:24,570 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:18:24,571 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:18:24,571 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:18:24,572 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:18:24,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:25,413 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:18:25,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:26,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:27,415 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:18:27,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:28,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:29,928 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:18:29,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:18:34,929 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:18:39,930 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:18:41,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:18:41,138 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:42,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:42,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:18:42,415 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:18:42,415 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:18:42,417 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:18:42,440 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:18:43,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:43,441 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:18:44,098 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:18:45,418 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:18:50,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:18:55,419 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:18:59,929 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:19:00,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:19:00,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:00,699 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:01,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:02,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:19:02,447 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:19:02,447 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:19:02,448 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:19:02,501 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:19:02,501 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:19:02,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:03,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:04,503 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:19:04,657 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:06,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:11,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:16,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:21,450 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:26,451 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:26,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:19:26,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:27,747 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:28,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:19:28,299 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:19:28,299 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:19:28,300 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:19:28,539 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:19:28,539 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:19:28,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:29,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:29,932 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:19:30,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:31,933 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:36,933 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:41,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:43,722 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:19:43,762 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:44,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:45,306 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:19:45,307 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:19:45,307 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:19:45,309 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:19:45,555 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:19:45,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:46,556 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:19:46,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:47,310 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:47,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:48,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:49,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:50,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:51,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:52,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:52,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:53,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:54,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:55,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:56,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:57,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:58,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:19:58,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:59,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:19:59,931 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:20:00,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:01,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:02,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:03,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:20:03,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:04,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:05,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:06,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:07,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:08,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:09,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:20:09,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:10,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:11,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:12,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:13,724 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:14,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:20:18,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:19,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:19,856 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:20:19,902 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:20:19,903 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:20:19,903 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:20:19,904 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:20:20,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:20,604 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:20:20,604 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:20:21,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:22,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:23,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:24,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:25,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:25,723 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:20:26,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:27,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:28,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:29,175 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:29,933 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:20:30,175 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:30,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:20:31,175 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:32,175 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:33,175 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:35,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:20:40,934 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:20:45,935 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:20:46,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:20:46,846 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:47,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:48,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:20:48,539 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:20:48,539 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:20:48,541 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:20:48,639 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:20:48,803 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:49,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:50,641 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:20:50,804 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:51,542 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:20:56,543 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:20:59,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:20:59,626 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:20:59,934 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:21:00,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:00,986 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:21:00,987 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:21:00,987 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:21:00,988 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:21:01,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:01,652 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:21:01,989 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:02,586 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:02,653 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:21:06,989 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:11,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:16,990 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:18,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:21:18,430 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:19,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:20,096 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:21:20,097 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:21:20,097 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:21:20,098 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:21:20,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:20,668 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:21:20,669 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:21:21,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:22,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:27,099 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:27,373 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:21:27,414 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:28,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:28,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:21:28,827 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:21:28,827 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:21:28,828 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:21:29,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:29,692 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:21:29,936 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:21:30,374 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:30,693 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:21:32,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:37,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:42,937 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:47,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:21:47,218 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:48,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:48,219 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:48,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:21:48,611 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:21:48,611 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:21:48,612 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:21:48,709 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:21:49,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:50,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:50,711 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:21:51,172 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:52,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:53,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:54,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:54,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:55,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:21:59,175 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:21:59,937 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:22:01,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:22:01,622 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:02,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:03,066 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:22:03,067 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:22:03,067 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:22:03,068 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:22:03,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:03,727 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:22:04,579 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:04,728 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:22:05,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:22:10,069 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:22:15,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:22:20,070 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:22:25,071 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:22:29,938 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:22:30,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:22:30,786 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:30,787 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:22:31,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:32,398 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:22:32,399 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:22:32,399 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:22:32,400 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:22:32,743 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:32,764 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:22:32,764 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:22:33,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:34,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:35,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:36,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:36,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:22:37,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:38,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:39,744 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:40,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:22:40,835 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:40,835 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:22:40,836 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:22:40,837 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:22:41,789 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:22:41,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:41,838 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:22:42,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:42,791 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:22:43,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:44,790 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:45,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:46,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:47,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:47,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:22:48,791 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:22:52,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:22:57,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:22:59,940 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:23:02,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:23:07,942 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:23:12,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:23:17,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:23:18,404 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:23:18,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:19,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:19,874 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:23:19,875 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:23:19,875 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:23:19,877 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:23:20,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:20,849 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:23:20,850 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:23:21,405 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:23,878 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:23:26,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:23:27,002 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:27,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:28,386 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:23:28,387 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:23:28,387 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:23:28,388 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:23:28,858 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:23:28,858 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:23:28,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:29,389 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:23:29,941 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:23:29,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:30,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:31,962 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:34,961 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:23:36,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:37,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:38,058 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:23:38,059 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:23:38,059 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:23:38,076 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:23:38,715 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:38,882 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:23:38,882 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:23:40,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:23:45,077 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:23:46,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:23:46,466 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:47,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:47,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:23:47,867 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:23:47,867 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:23:47,868 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:23:47,891 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:23:48,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:48,892 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:23:49,423 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:23:50,869 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:23:55,870 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:23:59,942 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:24:00,943 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:05,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:10,944 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:12,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:24:12,562 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:24:13,519 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:24:13,837 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:24:13,837 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:24:13,838 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:24:13,839 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:24:13,927 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:24:14,520 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:24:14,928 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:24:16,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:21,840 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:26,841 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:29,945 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:24:31,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:32,091 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:24:32,134 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:24:33,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:24:33,818 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:24:33,819 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:24:33,819 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:24:33,821 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:24:33,958 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:24:34,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:24:34,959 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:24:35,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:24:36,092 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:24:37,821 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:42,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:47,822 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:52,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:57,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:24:59,945 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:25:02,946 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:25:04,979 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:25:05,022 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:05,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:06,534 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:25:06,535 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:25:06,535 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:25:06,537 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:25:06,980 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:06,988 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:25:08,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:25:08,989 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:25:13,538 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:25:18,539 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:25:20,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:25:20,834 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:21,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:22,426 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:25:22,427 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:25:22,427 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:25:22,428 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:25:22,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:23,003 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:25:23,003 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:25:23,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:24,429 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:25:24,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:25,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:26,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:27,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:28,792 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:29,793 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:25:29,947 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:25:34,948 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:25:39,948 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:25:44,949 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:25:46,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:25:46,242 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:47,201 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:47,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:25:47,863 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:25:47,863 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:25:47,865 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:25:48,056 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:25:48,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:49,057 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:25:49,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:50,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:50,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:25:51,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:52,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:25:56,202 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:25:59,948 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:26:01,949 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:06,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:11,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:16,951 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:21,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:25,255 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:26:25,298 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:26,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:26,750 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:26:26,752 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:26:26,753 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:26:26,755 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:26:27,086 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:26:27,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:27,755 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:28,256 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:29,088 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:26:29,949 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:26:32,950 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:37,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:26:37,166 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:38,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:38,167 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:38,482 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:26:38,483 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:26:38,483 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:26:38,485 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:26:39,098 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:26:39,098 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:26:39,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:40,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:41,100 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:26:41,123 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:43,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:48,486 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:51,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:26:51,186 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:52,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:52,642 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:26:52,643 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:26:52,643 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:26:52,645 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:26:53,112 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:26:53,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:53,645 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:54,143 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:55,114 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:26:55,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:56,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:57,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:26:59,144 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:26:59,951 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:27:04,952 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:27:06,432 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:27:06,474 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:07,432 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:07,742 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:27:07,743 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:27:07,743 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:27:07,744 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:27:08,127 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:27:08,432 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:09,128 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:27:09,433 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:10,433 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:10,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:27:15,745 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:27:20,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:27:25,746 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:27:29,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:27:29,174 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:29,952 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:27:30,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:30,698 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:27:30,699 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:27:30,699 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:27:30,701 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:27:31,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:31,161 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:27:31,701 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:27:32,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:33,131 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:33,163 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:27:34,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:35,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:37,132 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:27:41,565 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:27:41,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:42,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:42,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:27:43,190 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:27:43,190 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:27:43,190 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:27:43,192 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:27:43,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:44,173 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:27:44,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:45,187 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:27:45,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:46,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:47,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:48,566 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:27:53,567 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:27:55,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:27:55,906 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:56,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:57,446 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:27:57,447 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:27:57,447 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:27:57,449 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:27:57,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:58,200 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:27:58,862 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:27:59,200 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:27:59,449 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:27:59,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:28:04,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:28:09,957 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:28:14,958 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:28:18,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: keepalive +2024-09-22 16:28:18,866 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:19,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:20,318 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:28:20,358 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 16:28:20,359 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:28:20,359 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:28:20,361 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:28:20,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:21,219 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:28:21,219 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:28:21,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:22,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:23,220 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:28:23,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:24,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:25,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:25,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:28:26,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:27,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:28,823 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:29,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:29,955 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:28:30,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:30,956 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:28:31,226 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:28:31,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:32,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:33,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:34,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:35,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:36,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:36,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:28:37,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:38,824 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:39,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:40,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:41,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:42,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:28:42,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:43,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:44,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:45,252 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:28:45,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:46,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:47,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:28:47,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:48,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:49,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:50,826 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:53,259 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:28:53,513 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:53,619 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:28:54,500 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:55,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:56,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:57,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:58,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:58,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:28:59,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:28:59,956 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:29:00,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:29:01,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:29:02,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:29:03,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:29:04,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:29:04,825 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:29:05,501 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:29:06,502 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:29:07,172 DEBUG SenderThread:1340627 [sender.py:send():379] send: telemetry +2024-09-22 16:29:07,173 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: server_info +2024-09-22 16:29:07,173 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: server_info +2024-09-22 16:29:07,379 DEBUG SenderThread:1340627 [sender.py:send():379] send: exit +2024-09-22 16:29:07,379 INFO SenderThread:1340627 [sender.py:send_exit():586] handling exit code: 0 +2024-09-22 16:29:07,379 INFO SenderThread:1340627 [sender.py:send_exit():588] handling runtime: 35739 +2024-09-22 16:29:07,381 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:29:07,381 INFO SenderThread:1340627 [sender.py:send_exit():594] send defer +2024-09-22 16:29:07,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:07,381 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 0 +2024-09-22 16:29:07,381 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:07,381 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-09-22 16:29:07,381 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 1 +2024-09-22 16:29:07,381 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:07,381 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 1 +2024-09-22 16:29:07,382 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:07,382 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-09-22 16:29:07,382 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 2 +2024-09-22 16:29:07,382 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:07,382 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 2 +2024-09-22 16:29:07,382 INFO HandlerThread:1340627 [system_monitor.py:finish():203] Stopping system monitor +2024-09-22 16:29:07,382 DEBUG SystemMonitor:1340627 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-22 16:29:07,382 INFO HandlerThread:1340627 [interfaces.py:finish():200] Joined cpu monitor +2024-09-22 16:29:07,382 DEBUG SystemMonitor:1340627 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-22 16:29:07,383 INFO HandlerThread:1340627 [interfaces.py:finish():200] Joined disk monitor +2024-09-22 16:29:08,270 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:29:08,601 INFO HandlerThread:1340627 [interfaces.py:finish():200] Joined gpu monitor +2024-09-22 16:29:08,601 INFO HandlerThread:1340627 [interfaces.py:finish():200] Joined memory monitor +2024-09-22 16:29:08,602 INFO HandlerThread:1340627 [interfaces.py:finish():200] Joined network monitor +2024-09-22 16:29:08,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-22 16:29:08,602 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:08,602 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-09-22 16:29:08,602 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 3 +2024-09-22 16:29:08,602 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:08,602 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 3 +2024-09-22 16:29:08,604 DEBUG SenderThread:1340627 [sender.py:send():379] send: stats +2024-09-22 16:29:08,604 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: poll_exit +2024-09-22 16:29:08,604 DEBUG SenderThread:1340627 [sender.py:send():379] send: history +2024-09-22 16:29:08,605 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: summary_record +2024-09-22 16:29:08,606 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:29:08,606 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:08,606 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-09-22 16:29:08,606 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 4 +2024-09-22 16:29:08,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:08,606 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 4 +2024-09-22 16:29:08,606 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:08,606 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-09-22 16:29:08,606 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 5 +2024-09-22 16:29:08,606 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:08,606 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 5 +2024-09-22 16:29:08,606 DEBUG SenderThread:1340627 [sender.py:send():379] send: summary +2024-09-22 16:29:08,607 INFO SenderThread:1340627 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 16:29:08,607 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:08,607 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-09-22 16:29:08,607 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 6 +2024-09-22 16:29:08,607 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:08,607 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 6 +2024-09-22 16:29:08,607 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:08,608 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-09-22 16:29:08,610 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 16:29:08,811 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 7 +2024-09-22 16:29:08,811 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:08,811 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 7 +2024-09-22 16:29:08,812 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:08,812 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-09-22 16:29:09,271 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:29:09,272 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/config.yaml +2024-09-22 16:29:09,272 INFO Thread-12 :1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:29:09,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-22 16:29:12,528 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 8 +2024-09-22 16:29:12,528 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: poll_exit +2024-09-22 16:29:12,528 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:12,528 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 8 +2024-09-22 16:29:12,528 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:12,528 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-09-22 16:29:12,529 INFO SenderThread:1340627 [job_builder.py:build():440] Attempting to build job artifact +2024-09-22 16:29:12,530 INFO SenderThread:1340627 [job_builder.py:_get_source_type():569] is repo sourced job +2024-09-22 16:29:12,530 WARNING SenderThread:1340627 [job_builder.py:_log_if_verbose():274] No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job +2024-09-22 16:29:12,530 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 9 +2024-09-22 16:29:12,530 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:12,530 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 9 +2024-09-22 16:29:12,530 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:12,530 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-09-22 16:29:12,530 INFO SenderThread:1340627 [dir_watcher.py:finish():358] shutting down directory watcher +2024-09-22 16:29:13,273 INFO SenderThread:1340627 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:29:13,273 INFO SenderThread:1340627 [dir_watcher.py:finish():388] scan: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files +2024-09-22 16:29:13,274 INFO SenderThread:1340627 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/conda-environment.yaml conda-environment.yaml +2024-09-22 16:29:13,274 INFO SenderThread:1340627 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log output.log +2024-09-22 16:29:13,274 INFO SenderThread:1340627 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-metadata.json wandb-metadata.json +2024-09-22 16:29:13,274 INFO SenderThread:1340627 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/requirements.txt requirements.txt +2024-09-22 16:29:13,274 INFO SenderThread:1340627 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json wandb-summary.json +2024-09-22 16:29:13,274 INFO SenderThread:1340627 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/config.yaml config.yaml +2024-09-22 16:29:13,277 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 10 +2024-09-22 16:29:13,279 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:13,280 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 10 +2024-09-22 16:29:13,280 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:13,280 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-09-22 16:29:13,280 INFO SenderThread:1340627 [file_pusher.py:finish():169] shutting down file pusher +2024-09-22 16:29:13,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-22 16:29:13,381 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: poll_exit +2024-09-22 16:29:13,796 INFO wandb-upload_3:1340627 [upload_job.py:push():130] Uploaded file ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json +2024-09-22 16:29:13,819 INFO wandb-upload_2:1340627 [upload_job.py:push():130] Uploaded file ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/requirements.txt +2024-09-22 16:29:13,836 INFO wandb-upload_4:1340627 [upload_job.py:push():130] Uploaded file ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/config.yaml +2024-09-22 16:29:13,847 INFO wandb-upload_0:1340627 [upload_job.py:push():130] Uploaded file ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/conda-environment.yaml +2024-09-22 16:29:13,924 INFO wandb-upload_1:1340627 [upload_job.py:push():130] Uploaded file ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/files/output.log +2024-09-22 16:29:14,124 INFO Thread-11 (_thread_body):1340627 [sender.py:transition_state():614] send defer: 11 +2024-09-22 16:29:14,124 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:14,124 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 11 +2024-09-22 16:29:14,125 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:14,125 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-09-22 16:29:14,125 INFO SenderThread:1340627 [file_pusher.py:join():175] waiting for file pusher +2024-09-22 16:29:14,125 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 12 +2024-09-22 16:29:14,125 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:14,125 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 12 +2024-09-22 16:29:14,125 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:14,125 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-09-22 16:29:14,125 INFO SenderThread:1340627 [file_stream.py:finish():601] file stream finish called +2024-09-22 16:29:14,380 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-22 16:29:14,795 INFO SenderThread:1340627 [file_stream.py:finish():605] file stream finish is done +2024-09-22 16:29:14,795 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 13 +2024-09-22 16:29:14,795 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: poll_exit +2024-09-22 16:29:14,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:14,796 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 13 +2024-09-22 16:29:14,796 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:14,796 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-09-22 16:29:14,796 INFO SenderThread:1340627 [sender.py:transition_state():614] send defer: 14 +2024-09-22 16:29:14,796 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: defer +2024-09-22 16:29:14,796 INFO HandlerThread:1340627 [handler.py:handle_request_defer():184] handle defer: 14 +2024-09-22 16:29:14,796 DEBUG SenderThread:1340627 [sender.py:send():379] send: final +2024-09-22 16:29:14,796 DEBUG SenderThread:1340627 [sender.py:send():379] send: footer +2024-09-22 16:29:14,796 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: defer +2024-09-22 16:29:14,796 INFO SenderThread:1340627 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-09-22 16:29:14,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-22 16:29:14,797 DEBUG SenderThread:1340627 [sender.py:send_request():406] send_request: poll_exit +2024-09-22 16:29:14,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 16:29:14,797 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: get_summary +2024-09-22 16:29:14,798 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: sampled_history +2024-09-22 16:29:14,799 DEBUG HandlerThread:1340627 [handler.py:handle_request():158] handle_request: shutdown +2024-09-22 16:29:14,799 INFO HandlerThread:1340627 [handler.py:finish():882] shutting down handler +2024-09-22 16:29:15,797 INFO WriterThread:1340627 [datastore.py:close():296] close: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/run-rptdqsqq.wandb +2024-09-22 16:29:15,797 INFO SenderThread:1340627 [sender.py:finish():1615] shutting down sender +2024-09-22 16:29:15,797 INFO SenderThread:1340627 [file_pusher.py:finish():169] shutting down file pusher +2024-09-22 16:29:15,797 INFO SenderThread:1340627 [file_pusher.py:join():175] waiting for file pusher