Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/run-20240922_063327-rptdqsqq/run-rptdqsqq.wandb filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,16 @@
+{
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

arguments.yaml

@@ -0,0 +1,51 @@
+data_cfgs:
+  eval_data_files: null
+  eval_datasets: null
+  eval_optional_args: []
+  eval_size: null
+  eval_split: null
+  eval_subset: null
+  eval_template: null
+  train_data_files: extracted_preference_10k_washed.json
+  train_datasets: /aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted
+  train_optional_args: []
+  train_size: null
+  train_split: train
+  train_subset: null
+  train_template: NExTQA_preference
+logger_cfgs:
+  cache_dir: null
+  log_project: align-anything
+  log_run_name: dpo
+  log_type: wandb
+  output_dir: ../outputs/dpo_tiv2t_10k_baseline
+  save_interval: 100000
+model_cfgs:
+  model_max_length: 4096
+  model_name_or_path: /aifs4su/yaodong/models/Qwen2-VL-7B-Instruct
+  trust_remote_code: true
+special_tokens: null
+train_cfgs:
+  adam_betas:
+  - 0.9
+  - 0.95
+  bf16: true
+  ds_cfgs: ds_z3_config.json
+  epochs: 3
+  eval_interval: 10
+  eval_strategy: epoch
+  fp16: false
+  freeze_language_model: false
+  freeze_mm_proj: false
+  freeze_vision_tower: true
+  gradient_accumulation_steps: 1.0
+  gradient_checkpointing: true
+  learning_rate: 1.0e-06
+  lr_scheduler_type: cosine
+  lr_warmup_ratio: 0.01
+  per_device_eval_batch_size: 1.0
+  per_device_train_batch_size: 1.0
+  regularization: 0.001
+  scale_coeff: 0.1
+  seed: 42
+  weight_decay: 0.0

config.json

@@ -0,0 +1,48 @@
+{
+  "_name_or_path": "/aifs4su/yaodong/models/Qwen2-VL-7B-Instruct",
+  "architectures": [
+    "Qwen2VLForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 128245,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "image_token_id": 151655,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2_vl",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "mrope_section": [
+      16,
+      24,
+      24
+    ],
+    "rope_type": "default",
+    "type": "default"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "in_chans": 3,
+    "model_type": "qwen2_vl",
+    "spatial_patch_size": 14
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 151657
+}

environ.txt

@@ -0,0 +1,45 @@
+CONDA_DEFAULT_ENV=hantao_new
+CONDA_EXE=/aifs4su/yaodong/miniconda3/bin/conda
+CONDA_PREFIX=/aifs4su/yaodong/miniconda3/envs/hantao_new
+CONDA_PREFIX_1=/aifs4su/yaodong/miniconda3
+CONDA_PYTHON_EXE=/aifs4su/yaodong/miniconda3/bin/python
+CROSS_RANK=0
+CROSS_SIZE=2
+CUDA_HOME=/aifs4su/yaodong/miniconda3/envs/hantao_new
+CUDA_MODULE_LOADING=LAZY
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1028/bus
+HOME=/home/yangyaodong
+KMP_DUPLICATE_LIB_OK=True
+KMP_INIT_AT_FORK=FALSE
+LANG=C.UTF-8
+LD_LIBRARY_PATH=/aifs4su/yaodong/miniconda3/envs/hantao_new/lib:/usr/mpi/gcc/openmpi-4.1.7a1/lib:/usr/local/cuda/lib64:/usr/local/cuda/lib64:/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
+LD_LIBRARY_PATH_modshare=/cm/shared/apps/slurm/current/lib64:1:/usr/mpi/gcc/openmpi-4.1.7a1/lib:1:/cm/shared/apps/slurm/current/lib64/slurm:1:/usr/local/cuda/lib64:1
+LD_RUN_PATH=/usr/mpi/gcc/openmpi-4.1.7a1/lib
+LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
+LOCAL_RANK=0
+LOCAL_SIZE=8
+LOGNAME=yangyaodong
+MASTER_ADDR=10.34.2.141
+MASTER_PORT=14989
+MOTD_SHOWN=pam
+OLDPWD=/aifs4su/yaodong/projects/hantao/dev_cham/align-anything/scripts
+PATH=/aifs4su/yaodong/google-cloud-sdk/bin:/aifs4su/yaodong/miniconda3/envs/hantao_new/bin:/aifs4su/yaodong/miniconda3/condabin:/usr/mpi/gcc/openmpi-4.1.7a1/bin:/usr/local/cuda/bin:/usr/lpp/mmfs/bin:/usr/local/cuda/bin:/opt/bin:/usr/local/cuda/bin:/usr/lpp/mmfs/bin:/cm/shared/apps/slurm/current/sbin:/cm/shared/apps/slurm/current/bin:/usr/local/cuda/bin:/opt/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/sbin:/usr/sbin:/cm/local/apps/environment-modules/4.5.3/bin
+PWD=/aifs4su/yaodong/projects/hantao/dev_cham/align-anything/scripts
+PYTHONHASHSEED=42
+PYTHONPATH=/aifs4su/yaodong/projects/hantao/dev_cham/align-anything/scripts:/aifs4su/yaodong/projects/hantao/dev_cham/align-anything
+RANK=0
+SHELL=/bin/bash
+SHLVL=0
+SSH_CLIENT=10.33.5.36 54644 22
+SSH_CONNECTION=10.33.5.36 54644 10.33.4.219 22
+TMUX_PANE=%0
+USER=yangyaodong
+WANDB_API_KEY=547f38af44135ca76a4f4eed9c8d135532da4960
+WANDB_SERVICE=2-1338204-tcp-localhost-59957
+WORLD_SIZE=16
+XDG_RUNTIME_DIR=/run/user/1028
+XDG_SESSION_CLASS=user
+XDG_SESSION_ID=626
+XDG_SESSION_TYPE=tty
+_=/aifs4su/yaodong/miniconda3/envs/hantao_tiv/bin/python3.11

preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 1003520,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "processor_class": "Qwen2VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "max_pixels": 1003520,
+    "min_pixels": 3136
+  },
+  "temporal_patch_size": 2
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23508df701cbdb806cf6414699da63f6dd30345623489cea766f23284c330cc0
+size 16577151950

script.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 PKU-Alignment Team. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+export TRITON_CACHE_DIR="/aifs4su/yaodong/cache/triton"
+# You can replace it with a local model path
+MODEL_NAME_OR_PATH="/aifs4su/yaodong/models/Qwen2-VL-7B-Instruct"
+# You can replace it with a local dataset path
+TRAIN_DATASETS="/aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted"
+EVAL_DATASETS="/aifs4su/yaodong/datasets/ShareGPTVideo/extracted/split"
+# You can replace it with a new path
+OUTPUT_DIR="../outputs/dpo_tiv2t_10k_baseline"
+# For wandb online logging
+export WANDB_API_KEY="547f38af44135ca76a4f4eed9c8d135532da4960"
+# Source the setup script
+source ./setup.sh
+
+export CUDA_HOME=$CONDA_PREFIX
+
+# Execute deepspeed command
+deepspeed \
+	--hostfile ./host9 \
+	--master_port ${MASTER_PORT} \
+	--module align_anything.trainers.tiv_to_t.dpo \
+	--model_name_or_path ${MODEL_NAME_OR_PATH} \
+	--train_datasets ${TRAIN_DATASETS} \
+	--output_dir ${OUTPUT_DIR} \
+	--train_data_files extracted_preference_10k_washed.json \
+	--train_template NExTQA_preference \
+	--train_split 'train' \
+	--per_device_train_batch_size 1 \
+	--per_device_eval_batch_size 1 \
+	--gradient_accumulation_steps 1

special_tokens_map.json

@@ -0,0 +1,45 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,160 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "128244": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": "<s>",
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "processor_class": "Qwen2VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": "<unk>"
+}

wandb/debug.log

@@ -0,0 +1,36 @@
+2024-09-22 06:33:27,559 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Configure stats pid to 1338204
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Loading settings from /home/yangyaodong/.config/wandb/settings
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Loading settings from /aifs4su/yaodong/projects/hantao/dev_cham/align-anything/scripts/wandb/settings
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***'}
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-09-22 06:33:27,560 WARNING MainThread:1338204 [wandb_setup.py:_flush():76] Could not find program at -m align_anything.trainers.tiv_to_t.dpo
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.tiv_to_t.dpo'}
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:_log_setup():529] Logging user logs to ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/logs/debug.log
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:_log_setup():530] Logging internal logs to ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/logs/debug-internal.log
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:init():569] calling init triggers
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1.0, 'per_device_eval_batch_size': 1.0, 'gradient_accumulation_steps': 1.0, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.01, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'regularization': 0.001, 'scale_coeff': 0.1, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False}, 'data_cfgs': {'train_datasets': '/aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted', 'train_template': 'NExTQA_preference', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'extracted_preference_10k_washed.json', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'dpo', 'output_dir': '../outputs/dpo_tiv2t_10k_baseline', 'cache_dir': None, 'save_interval': 100000}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/models/Qwen2-VL-7B-Instruct', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:init():619] starting backend
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:init():623] setting up manager
+2024-09-22 06:33:27,561 INFO    MainThread:1338204 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-09-22 06:33:27,564 INFO    MainThread:1338204 [wandb_init.py:init():631] backend started and connected
+2024-09-22 06:33:27,566 INFO    MainThread:1338204 [wandb_init.py:init():720] updated telemetry
+2024-09-22 06:33:27,587 INFO    MainThread:1338204 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-09-22 06:33:28,008 INFO    MainThread:1338204 [wandb_run.py:_on_init():2435] communicating current version
+2024-09-22 06:33:28,205 INFO    MainThread:1338204 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.18.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-09-22 06:33:28,205 INFO    MainThread:1338204 [wandb_init.py:init():804] starting run threads in backend
+2024-09-22 06:33:34,310 INFO    MainThread:1338204 [wandb_run.py:_console_start():2413] atexit reg
+2024-09-22 06:33:34,310 INFO    MainThread:1338204 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-09-22 06:33:34,310 INFO    MainThread:1338204 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-09-22 06:33:34,310 INFO    MainThread:1338204 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-09-22 06:33:34,313 INFO    MainThread:1338204 [wandb_init.py:init():847] run started, returning control to user process
+2024-09-22 16:29:07,169 INFO    MainThread:1338204 [wandb_run.py:_finish():2107] finishing run htlou/align-anything/rptdqsqq
+2024-09-22 16:29:07,171 INFO    MainThread:1338204 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0
+2024-09-22 16:29:07,172 INFO    MainThread:1338204 [wandb_run.py:_restore():2352] restore
+2024-09-22 16:29:07,172 INFO    MainThread:1338204 [wandb_run.py:_restore():2358] restore done
+2024-09-22 16:29:15,801 INFO    MainThread:1338204 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-09-22 16:29:15,802 INFO    MainThread:1338204 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-09-22 16:29:15,809 INFO    MainThread:1338204 [wandb_run.py:_footer_sync_info():3975] logging synced files

wandb/run-20240922_051551-1efs7bbc/files/conda-environment.yaml

@@ -0,0 +1,504 @@
+name: hantao_new
+channels:
+  - nvidia/label/cuda-12.1.0
+  - nvidia
+  - pytorch
+  - huggingface
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - aom=3.9.1=hac33072_0
+  - asttokens=2.4.1=pyhd8ed1ab_0
+  - blas=1.0=mkl
+  - brotli-python=1.1.0=py311hb755f60_1
+  - bzip2=1.0.8=h4bc722e_7
+  - ca-certificates=2024.7.4=hbcca054_0
+  - cairo=1.18.0=hebfffa5_3
+  - certifi=2024.7.4=pyhd8ed1ab_0
+  - cffi=1.16.0=py311hb3a22ac_0
+  - charset-normalizer=3.3.2=pyhd8ed1ab_0
+  - comm=0.2.2=pyhd8ed1ab_0
+  - cuda=12.1.0=0
+  - cuda-cccl=12.1.55=0
+  - cuda-command-line-tools=12.1.0=0
+  - cuda-compiler=12.1.0=0
+  - cuda-cudart=12.1.55=0
+  - cuda-cudart-dev=12.1.55=0
+  - cuda-cudart-static=12.1.55=0
+  - cuda-cuobjdump=12.1.55=0
+  - cuda-cupti=12.1.62=0
+  - cuda-cupti-static=12.1.62=0
+  - cuda-cuxxfilt=12.1.55=0
+  - cuda-demo-suite=12.1.55=0
+  - cuda-documentation=12.1.55=0
+  - cuda-driver-dev=12.1.55=0
+  - cuda-gdb=12.1.55=0
+  - cuda-libraries=12.1.0=0
+  - cuda-libraries-dev=12.1.0=0
+  - cuda-libraries-static=12.1.0=0
+  - cuda-nsight=12.1.55=0
+  - cuda-nsight-compute=12.1.0=0
+  - cuda-nvcc=12.1.66=0
+  - cuda-nvdisasm=12.1.55=0
+  - cuda-nvml-dev=12.1.55=0
+  - cuda-nvprof=12.1.55=0
+  - cuda-nvprune=12.1.55=0
+  - cuda-nvrtc=12.1.55=0
+  - cuda-nvrtc-dev=12.1.55=0
+  - cuda-nvrtc-static=12.1.55=0
+  - cuda-nvtx=12.1.66=0
+  - cuda-nvvp=12.1.55=0
+  - cuda-opencl=12.1.56=0
+  - cuda-opencl-dev=12.1.56=0
+  - cuda-profiler-api=12.1.55=0
+  - cuda-runtime=12.1.0=0
+  - cuda-sanitizer-api=12.1.55=0
+  - cuda-toolkit=12.1.0=0
+  - cuda-tools=12.1.0=0
+  - cuda-visual-tools=12.1.0=0
+  - dav1d=1.2.1=hd590300_0
+  - debugpy=1.8.2=py311h4332511_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - exceptiongroup=1.2.2=pyhd8ed1ab_0
+  - executing=2.0.1=pyhd8ed1ab_0
+  - expat=2.6.2=h59595ed_0
+  - ffmpeg=7.0.1=gpl_h9be9148_104
+  - filelock=3.15.4=pyhd8ed1ab_0
+  - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
+  - font-ttf-inconsolata=3.000=h77eed37_0
+  - font-ttf-source-code-pro=2.038=h77eed37_0
+  - font-ttf-ubuntu=0.83=h77eed37_2
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-conda-ecosystem=1=0
+  - fonts-conda-forge=1=0
+  - freetype=2.12.1=h267a509_2
+  - fribidi=1.0.10=h36c2ea0_0
+  - gds-tools=1.6.0.25=0
+  - gettext=0.22.5=h59595ed_2
+  - gettext-tools=0.22.5=h59595ed_2
+  - git-lfs=3.5.1=ha770c72_0
+  - gmp=6.3.0=hac33072_2
+  - gmpy2=2.1.5=py311hc4f1f91_1
+  - gnutls=3.7.9=hb077bed_0
+  - graphite2=1.3.13=h59595ed_1003
+  - h2=4.1.0=pyhd8ed1ab_0
+  - harfbuzz=9.0.0=hda332d3_1
+  - hpack=4.0.0=pyh9f0ad1d_0
+  - hyperframe=6.0.1=pyhd8ed1ab_0
+  - icu=75.1=he02047a_0
+  - idna=3.7=pyhd8ed1ab_0
+  - importlib-metadata=8.2.0=pyha770c72_0
+  - importlib_metadata=8.2.0=hd8ed1ab_0
+  - intel-openmp=2022.0.1=h06a4308_3633
+  - ipykernel=6.29.5=pyh3099207_0
+  - ipython=8.26.0=pyh707e725_0
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jinja2=3.1.4=pyhd8ed1ab_0
+  - jupyter_client=8.6.2=pyhd8ed1ab_0
+  - jupyter_core=5.7.2=py311h38be061_0
+  - keyutils=1.6.1=h166bdaf_0
+  - krb5=1.21.3=h659f571_0
+  - lame=3.100=h166bdaf_1003
+  - lcms2=2.16=hb7c19ff_0
+  - ld_impl_linux-64=2.40=hf3520f5_7
+  - lerc=4.0.0=h27087fc_0
+  - libabseil=20240116.2=cxx17_he02047a_1
+  - libasprintf=0.22.5=h661eb56_2
+  - libasprintf-devel=0.22.5=h661eb56_2
+  - libass=0.17.1=h39113c1_2
+  - libblas=3.9.0=16_linux64_mkl
+  - libcblas=3.9.0=16_linux64_mkl
+  - libcublas=12.1.0.26=0
+  - libcublas-dev=12.1.0.26=0
+  - libcublas-static=12.1.0.26=0
+  - libcufft=11.0.2.4=0
+  - libcufft-dev=11.0.2.4=0
+  - libcufft-static=11.0.2.4=0
+  - libcufile=1.6.0.25=0
+  - libcufile-dev=1.6.0.25=0
+  - libcufile-static=1.6.0.25=0
+  - libcurand=10.3.2.56=0
+  - libcurand-dev=10.3.2.56=0
+  - libcurand-static=10.3.2.56=0
+  - libcusolver=11.4.4.55=0
+  - libcusolver-dev=11.4.4.55=0
+  - libcusolver-static=11.4.4.55=0
+  - libcusparse=12.0.2.55=0
+  - libcusparse-dev=12.0.2.55=0
+  - libcusparse-static=12.0.2.55=0
+  - libdeflate=1.20=hd590300_0
+  - libdrm=2.4.122=h4ab18f5_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libexpat=2.6.2=h59595ed_0
+  - libffi=3.4.2=h7f98852_5
+  - libgcc-ng=14.1.0=h77fa898_0
+  - libgettextpo=0.22.5=h59595ed_2
+  - libgettextpo-devel=0.22.5=h59595ed_2
+  - libglib=2.80.3=h8a4344b_1
+  - libgomp=14.1.0=h77fa898_0
+  - libhwloc=2.11.1=default_hecaa2ac_1000
+  - libiconv=1.17=hd590300_2
+  - libidn2=2.3.7=hd590300_0
+  - libjpeg-turbo=3.0.0=hd590300_1
+  - liblapack=3.9.0=16_linux64_mkl
+  - libnpp=12.0.2.50=0
+  - libnpp-dev=12.0.2.50=0
+  - libnpp-static=12.0.2.50=0
+  - libnsl=2.0.1=hd590300_0
+  - libnvjitlink=12.1.105=0
+  - libnvjitlink-dev=12.1.55=0
+  - libnvjpeg=12.1.0.39=0
+  - libnvjpeg-dev=12.1.0.39=0
+  - libnvjpeg-static=12.1.0.39=0
+  - libnvvm-samples=12.1.55=0
+  - libopenvino=2024.2.0=h2da1b83_1
+  - libopenvino-auto-batch-plugin=2024.2.0=hb045406_1
+  - libopenvino-auto-plugin=2024.2.0=hb045406_1
+  - libopenvino-hetero-plugin=2024.2.0=h5c03a75_1
+  - libopenvino-intel-cpu-plugin=2024.2.0=h2da1b83_1
+  - libopenvino-intel-gpu-plugin=2024.2.0=h2da1b83_1
+  - libopenvino-intel-npu-plugin=2024.2.0=he02047a_1
+  - libopenvino-ir-frontend=2024.2.0=h5c03a75_1
+  - libopenvino-onnx-frontend=2024.2.0=h07e8aee_1
+  - libopenvino-paddle-frontend=2024.2.0=h07e8aee_1
+  - libopenvino-pytorch-frontend=2024.2.0=he02047a_1
+  - libopenvino-tensorflow-frontend=2024.2.0=h39126c6_1
+  - libopenvino-tensorflow-lite-frontend=2024.2.0=he02047a_1
+  - libopus=1.3.1=h7f98852_1
+  - libpciaccess=0.18=hd590300_0
+  - libpng=1.6.43=h2797004_0
+  - libprotobuf=4.25.3=h08a7969_0
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsqlite=3.46.0=hde9e2c9_0
+  - libstdcxx-ng=14.1.0=hc0a3c3a_0
+  - libtasn1=4.19.0=h166bdaf_0
+  - libtiff=4.6.0=h1dd3fc0_3
+  - libunistring=0.9.10=h7f98852_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libva=2.22.0=hb711507_0
+  - libvpx=1.14.1=hac33072_0
+  - libwebp-base=1.4.0=hd590300_0
+  - libxcb=1.16=hd590300_0
+  - libxcrypt=4.4.36=hd590300_1
+  - libxml2=2.12.7=he7c6b58_4
+  - libzlib=1.3.1=h4ab18f5_1
+  - llvm-openmp=15.0.7=h0cdce71_0
+  - markupsafe=2.1.5=py311h459d7ec_0
+  - matplotlib-inline=0.1.7=pyhd8ed1ab_0
+  - mkl=2022.1.0=hc2b9512_224
+  - mpc=1.3.1=hfe3b2da_0
+  - mpfr=4.2.1=h38ae2d0_2
+  - mpmath=1.3.0=pyhd8ed1ab_0
+  - ncurses=6.5=h59595ed_0
+  - nest-asyncio=1.6.0=pyhd8ed1ab_0
+  - nettle=3.9.1=h7ab15ed_0
+  - networkx=3.3=pyhd8ed1ab_1
+  - nsight-compute=2023.1.0.15=0
+  - ocl-icd=2.3.2=hd590300_1
+  - openh264=2.4.1=h59595ed_0
+  - openjpeg=2.5.2=h488ebb8_0
+  - openssl=3.3.1=h4bc722e_2
+  - p11-kit=0.24.1=hc5aa10d_0
+  - packaging=24.1=pyhd8ed1ab_0
+  - parso=0.8.4=pyhd8ed1ab_0
+  - pcre2=10.44=h0f59acf_0
+  - pexpect=4.9.0=pyhd8ed1ab_0
+  - pickleshare=0.7.5=py_1003
+  - pillow=10.4.0=py311h82a398c_0
+  - pip=24.0=pyhd8ed1ab_0
+  - pixman=0.43.2=h59595ed_0
+  - prompt-toolkit=3.0.47=pyha770c72_0
+  - psutil=6.0.0=py311h331c9d8_0
+  - pthread-stubs=0.4=h36c2ea0_1001
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pugixml=1.14=h59595ed_0
+  - pure_eval=0.2.3=pyhd8ed1ab_0
+  - pycparser=2.22=pyhd8ed1ab_0
+  - pygments=2.18.0=pyhd8ed1ab_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - python=3.11.9=hb806964_0_cpython
+  - python_abi=3.11=4_cp311
+  - pytorch=2.4.0=py3.11_cuda12.1_cudnn9.1.0_0
+  - pytorch-cuda=12.1=ha16c6d3_5
+  - pytorch-mutex=1.0=cuda
+  - pyyaml=6.0.1=py311h459d7ec_1
+  - pyzmq=26.0.3=py311h08a0b41_0
+  - readline=8.2=h8228510_1
+  - requests=2.32.3=pyhd8ed1ab_0
+  - setuptools=71.0.4=pyhd8ed1ab_0
+  - six=1.16.0=pyh6c4a22f_0
+  - snappy=1.2.1=ha2e4443_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - svt-av1=2.1.2=hac33072_0
+  - sympy=1.13.0=pypyh2585a3b_103
+  - tbb=2021.12.0=h434a139_3
+  - tk=8.6.13=noxft_h4845f30_101
+  - torchaudio=2.4.0=py311_cu121
+  - torchtriton=3.0.0=py311
+  - tornado=6.4.1=py311h331c9d8_0
+  - traitlets=5.14.3=pyhd8ed1ab_0
+  - typing_extensions=4.12.2=pyha770c72_0
+  - urllib3=2.2.2=pyhd8ed1ab_1
+  - wayland=1.23.0=h5291e77_0
+  - wayland-protocols=1.36=hd8ed1ab_0
+  - wcwidth=0.2.13=pyhd8ed1ab_0
+  - wheel=0.43.0=pyhd8ed1ab_1
+  - x264=1!164.3095=h166bdaf_2
+  - x265=3.5=h924138e_3
+  - xorg-fixesproto=5.0=h7f98852_1002
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.9=hb711507_1
+  - xorg-libxau=1.0.11=hd590300_0
+  - xorg-libxdmcp=1.1.3=h7f98852_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxfixes=5.0.3=h7f98852_1004
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xproto=7.0.31=h7f98852_1007
+  - xz=5.2.6=h166bdaf_0
+  - yaml=0.2.5=h7f98852_2
+  - zeromq=4.3.5=h75354e8_4
+  - zipp=3.19.2=pyhd8ed1ab_0
+  - zlib=1.3.1=h4ab18f5_1
+  - zstandard=0.23.0=py311h5cd10c7_0
+  - zstd=1.5.6=ha6fb4c9_0
+  - pip:
+      - about-time==4.2.1
+      - absl-py==2.1.0
+      - accelerate==0.33.0
+      - aiofiles==23.2.1
+      - aiohttp==3.9.5
+      - aiohttp-retry==2.8.3
+      - aiosignal==1.3.1
+      - align-anything==0.0.1.dev0
+      - alive-progress==3.1.5
+      - amqp==5.2.0
+      - annotated-types==0.7.0
+      - antlr4-python3-runtime==4.9.3
+      - anyio==4.4.0
+      - appdirs==1.4.4
+      - asyncssh==2.16.0
+      - atpublic==5.0
+      - attrs==23.2.0
+      - audioread==3.0.1
+      - av==13.0.0
+      - backoff==2.2.1
+      - billiard==4.2.0
+      - boto3==1.35.9
+      - botocore==1.35.9
+      - cachetools==5.4.0
+      - celery==5.4.0
+      - cfgv==3.4.0
+      - click==8.1.7
+      - click-didyoumean==0.3.1
+      - click-plugins==1.1.1
+      - click-repl==0.3.0
+      - cloudpickle==3.0.0
+      - cmake==3.30.1
+      - colorama==0.4.6
+      - commonmark==0.9.1
+      - configobj==5.0.8
+      - contourpy==1.2.1
+      - cryptography==43.0.0
+      - cycler==0.12.1
+      - dacite==1.6.0
+      - dagshub==0.3.35
+      - dagshub-annotation-converter==0.1.0
+      - dataclasses-json==0.6.7
+      - datasets==2.20.0
+      - deepspeed==0.14.4
+      - dictdiffer==0.9.0
+      - diffusers==0.29.2
+      - dill==0.3.8
+      - diskcache==5.6.3
+      - distlib==0.3.8
+      - distro==1.9.0
+      - dnspython==2.6.1
+      - docker-pycreds==0.4.0
+      - dpath==2.2.0
+      - dulwich==0.22.1
+      - dvc==3.55.1
+      - dvc-data==3.16.5
+      - dvc-http==2.32.0
+      - dvc-objects==5.1.0
+      - dvc-render==1.0.2
+      - dvc-studio-client==0.21.0
+      - dvc-task==0.4.0
+      - email-validator==2.2.0
+      - entrypoints==0.4
+      - fastapi==0.111.1
+      - fastapi-cli==0.0.4
+      - ffmpy==0.3.3
+      - flatten-dict==0.4.2
+      - flufl-lock==8.1.0
+      - fonttools==4.53.1
+      - frozenlist==1.4.1
+      - fsspec==2024.5.0
+      - funcy==2.0
+      - fusepy==3.0.1
+      - gitdb==4.0.11
+      - gitpython==3.1.43
+      - gql==3.5.0
+      - gradio==4.39.0
+      - gradio-client==1.1.1
+      - gradio-multimodalchatbot==0.0.1
+      - grandalf==0.8
+      - grapheme==0.6.0
+      - graphql-core==3.2.3
+      - grpcio==1.65.1
+      - gto==1.7.1
+      - h11==0.14.0
+      - hjson==3.1.0
+      - httpcore==0.16.3
+      - httptools==0.6.1
+      - httpx==0.23.3
+      - huggingface-hub==0.24.3
+      - hydra-core==1.3.2
+      - identify==2.6.0
+      - importlib-resources==6.4.0
+      - iniconfig==2.0.0
+      - interegular==0.3.3
+      - iterative-telemetry==0.0.8
+      - jmespath==1.0.1
+      - joblib==1.4.2
+      - jsonschema==4.23.0
+      - jsonschema-specifications==2023.12.1
+      - kiwisolver==1.4.5
+      - kombu==5.4.0
+      - lark==1.1.9
+      - lazy-loader==0.4
+      - librosa==0.10.2.post1
+      - llvmlite==0.43.0
+      - lm-format-enforcer==0.10.3
+      - lxml==5.3.0
+      - markdown==3.6
+      - markdown-it-py==3.0.0
+      - marshmallow==3.22.0
+      - matplotlib==3.9.1
+      - mdurl==0.1.2
+      - mmsg==0.1.dev14+gbd2b056.d20240803
+      - msgpack==1.0.8
+      - multidict==6.0.5
+      - multiprocess==0.70.16
+      - mypy-extensions==1.0.0
+      - ninja==1.11.1.1
+      - nodeenv==1.9.1
+      - numba==0.60.0
+      - numpy==1.26.4
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu12==8.9.2.26
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-ml-py==12.535.161
+      - nvidia-nccl-cu12==2.20.5
+      - nvidia-nvjitlink-cu12==12.5.82
+      - nvidia-nvtx-cu12==12.1.105
+      - nvitop==1.3.2
+      - omegaconf==2.3.0
+      - openai==1.37.1
+      - optree==0.12.1
+      - orjson==3.10.6
+      - outlines==0.0.47.dev37+g26e2934
+      - pandas==2.2.2
+      - pathspec==0.12.1
+      - pathvalidate==3.0.0
+      - peft==0.12.0
+      - platformdirs==3.11.0
+      - pluggy==1.5.0
+      - pooch==1.8.2
+      - pre-commit==3.8.0
+      - prometheus-client==0.20.0
+      - prometheus-fastapi-instrumentator==7.0.0
+      - protobuf==4.25.4
+      - py-cpuinfo==9.0.0
+      - pyairports==2.1.1
+      - pyarrow==17.0.0
+      - pyarrow-hotfix==0.6
+      - pycountry==24.6.1
+      - pydantic==2.8.2
+      - pydantic-core==2.20.1
+      - pydot==3.0.1
+      - pydub==0.25.1
+      - pygit2==1.15.1
+      - pygtrie==2.5.0
+      - pyparsing==3.1.2
+      - pytest==8.3.2
+      - python-dateutil==2.9.0.post0
+      - python-dotenv==1.0.1
+      - python-multipart==0.0.9
+      - pytz==2024.1
+      - ranger-fm==1.9.3
+      - ray==2.34.0
+      - referencing==0.35.1
+      - regex==2024.7.24
+      - requests-toolbelt==1.0.0
+      - rfc3986==1.5.0
+      - rich==13.1.0
+      - rpds-py==0.19.1
+      - ruamel-yaml==0.18.6
+      - ruamel-yaml-clib==0.2.8
+      - ruff==0.5.5
+      - s3transfer==0.10.2
+      - safetensors==0.4.3
+      - scikit-learn==1.5.1
+      - scipy==1.14.0
+      - scmrepo==3.3.7
+      - semantic-version==2.10.0
+      - semver==3.0.2
+      - sentencepiece==0.2.0
+      - sentry-sdk==2.11.0
+      - setproctitle==1.3.3
+      - shellingham==1.5.4
+      - shortuuid==1.0.13
+      - shtab==1.7.1
+      - smmap==5.0.1
+      - sniffio==1.3.1
+      - soundfile==0.12.1
+      - soxr==0.4.0
+      - sqltrie==0.11.1
+      - starlette==0.37.2
+      - tabulate==0.9.0
+      - tenacity==8.2.3
+      - tensorboard==2.17.0
+      - tensorboard-data-server==0.7.2
+      - term-image==0.7.2
+      - termcolor==2.4.0
+      - threadpoolctl==3.5.0
+      - tiktoken==0.7.0
+      - tokenizers==0.19.1
+      - tomlkit==0.12.0
+      - torch==2.3.1
+      - torchvision==0.18.1
+      - tqdm==4.66.4
+      - transformers==4.44.0.dev0
+      - treelib==1.6.4
+      - triton==2.3.1
+      - typer==0.12.3
+      - typing-inspect==0.9.0
+      - tzdata==2024.1
+      - uvicorn==0.30.3
+      - uvloop==0.19.0
+      - vine==5.1.0
+      - virtualenv==20.26.3
+      - vllm==0.5.3.post1
+      - vllm-flash-attn==2.5.9.post1
+      - voluptuous==0.15.2
+      - wandb==0.17.5
+      - watchfiles==0.22.0
+      - websockets==11.0.3
+      - werkzeug==3.0.3
+      - xformers==0.0.27
+      - xxhash==3.4.1
+      - yarl==1.9.4
+      - zc-lockfile==3.0.post1
+prefix: /aifs4su/yaodong/miniconda3/envs/hantao_new

wandb/run-20240922_051551-1efs7bbc/files/config.yaml

@@ -0,0 +1,111 @@
+wandb_version: 1
+
+train_cfgs:
+  desc: null
+  value:
+    ds_cfgs: ds_z3_config.json
+    epochs: 3
+    seed: 42
+    per_device_train_batch_size: 1.0
+    per_device_eval_batch_size: 1.0
+    gradient_accumulation_steps: 1.0
+    gradient_checkpointing: true
+    learning_rate: 1.0e-06
+    lr_scheduler_type: cosine
+    lr_warmup_ratio: 0.01
+    weight_decay: 0.0
+    adam_betas:
+    - 0.9
+    - 0.95
+    bf16: true
+    fp16: false
+    eval_strategy: epoch
+    eval_interval: 10
+    regularization: 0.001
+    scale_coeff: 0.1
+    freeze_mm_proj: false
+    freeze_vision_tower: true
+    freeze_language_model: false
+data_cfgs:
+  desc: null
+  value:
+    train_datasets: /aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted
+    train_template: NExTQA_preference
+    train_size: null
+    train_split: train
+    train_subset: null
+    train_data_files: extracted_preference_10k_washed.json
+    train_optional_args: []
+    eval_datasets: null
+    eval_template: null
+    eval_size: null
+    eval_split: null
+    eval_subset: null
+    eval_data_files: null
+    eval_optional_args: []
+logger_cfgs:
+  desc: null
+  value:
+    log_type: wandb
+    log_project: align-anything
+    log_run_name: dpo
+    output_dir: ../outputs/dpo_tiv2t_10k_baseline
+    cache_dir: null
+    save_interval: 100000
+model_cfgs:
+  desc: null
+  value:
+    model_name_or_path: /aifs4su/yaodong/models/Qwen2-VL-7B-Instruct
+    trust_remote_code: true
+    model_max_length: 4096
+special_tokens:
+  desc: null
+  value: null
+_wandb:
+  desc: null
+  value:
+    python_version: 3.11.9
+    cli_version: 0.17.5
+    framework: huggingface
+    huggingface_version: 4.45.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1726953351
+    t:
+      1:
+      - 1
+      - 5
+      - 11
+      - 41
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 83
+      - 98
+      - 105
+      2:
+      - 1
+      - 5
+      - 11
+      - 41
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 83
+      - 98
+      - 105
+      3:
+      - 13
+      - 16
+      - 23
+      - 61
+      4: 3.11.9
+      5: 0.17.5
+      6: 4.45.0.dev0
+      8:
+      - 5
+      13: linux-x86_64

wandb/run-20240922_051551-1efs7bbc/files/output.log

@@ -0,0 +1,88 @@
+
+***** Running training *****
+
+Training 1/3 epoch:   0%|          | 0/1815 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...
+/aifs4su/yaodong/miniconda3/envs/hantao_tiv/lib/python3.11/site-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
+  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.6823):   0%|          | 9/1815 [02:56<9:53:17, 19.71s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.6167):   1%|          | 19/1815 [06:28<11:02:22, 22.13s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.9150):   2%|▏         | 29/1815 [09:48<8:59:30, 18.12s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.0755):   2%|▏         | 40/1815 [13:41<10:30:16, 21.31s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.8264):   3%|▎         | 50/1815 [17:17<9:36:08, 19.59s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.4644):   3%|▎         | 60/1815 [20:59<10:18:16, 21.14s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.0547):   4%|▍         | 70/1815 [24:46<10:25:39, 21.51s/it]
+
+
+
+
+
+

wandb/run-20240922_051551-1efs7bbc/files/requirements.txt

@@ -0,0 +1,325 @@
+Brotli==1.1.0
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Werkzeug==3.0.3
+about-time==4.2.1
+absl-py==2.1.0
+accelerate==0.34.0
+aiofiles==23.2.1
+aiohttp-retry==2.8.3
+aiohttp==3.9.5
+aiosignal==1.3.1
+align-anything==0.0.1.dev0
+align-anything==0.0.1.dev0
+alive-progress==3.1.5
+amqp==5.2.0
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+appdirs==1.4.4
+asttokens==2.4.1
+asyncssh==2.16.0
+atpublic==5.0
+attrs==23.2.0
+audioread==3.0.1
+autocommand==2.2.2
+backoff==2.2.1
+backports.tarfile==1.2.0
+billiard==4.2.0
+boto3==1.35.9
+botocore==1.35.9
+cachetools==5.4.0
+celery==5.4.0
+certifi==2024.7.4
+cffi==1.16.0
+cfgv==3.4.0
+charset-normalizer==3.3.2
+click-didyoumean==0.3.1
+click-plugins==1.1.1
+click-repl==0.3.0
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorama==0.4.6
+comm==0.2.2
+commonmark==0.9.1
+configobj==5.0.8
+contourpy==1.2.1
+cryptography==43.0.0
+cycler==0.12.1
+dacite==1.6.0
+dagshub-annotation-converter==0.1.0
+dagshub==0.3.35
+dataclasses-json==0.6.7
+datasets==2.20.0
+debugpy==1.8.2
+decorator==4.4.2
+deepspeed==0.14.4
+dictdiffer==0.9.0
+diffusers==0.29.2
+dill==0.3.8
+diskcache==5.6.3
+distlib==0.3.8
+distro==1.9.0
+dnspython==2.6.1
+docker-pycreds==0.4.0
+docstring_parser==0.16
+dpath==2.2.0
+dulwich==0.22.1
+dvc-data==3.16.5
+dvc-http==2.32.0
+dvc-objects==5.1.0
+dvc-render==1.0.2
+dvc-studio-client==0.21.0
+dvc-task==0.4.0
+dvc==3.55.1
+einops==0.8.0
+email_validator==2.2.0
+entrypoints==0.4
+exceptiongroup==1.2.2
+executing==2.0.1
+fastapi-cli==0.0.4
+fastapi==0.111.1
+ffmpy==0.3.3
+filelock==3.15.4
+fire==0.6.0
+flatten-dict==0.4.2
+flufl.lock==8.1.0
+fonttools==4.53.1
+frozenlist==1.4.1
+fsspec==2024.5.0
+funcy==2.0
+fusepy==3.0.1
+gitdb==4.0.11
+gmpy2==2.1.5
+gql==3.5.0
+gradio==4.39.0
+gradio_client==1.1.1
+gradio_multimodalchatbot==0.0.1
+grandalf==0.8
+grapheme==0.6.0
+graphql-core==3.2.3
+grpcio==1.65.1
+gto==1.7.1
+h11==0.14.0
+h2==4.1.0
+hjson==3.1.0
+hpack==4.0.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.2
+huggingface-hub==0.24.3
+hydra-core==1.3.2
+hyperframe==6.0.1
+identify==2.6.0
+idna==3.7
+imageio-ffmpeg==0.5.1
+imageio==2.35.1
+importlib_metadata==8.0.0
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+importlib_resources==6.4.0
+inflect==7.3.1
+iniconfig==2.0.0
+interegular==0.3.3
+ipykernel==6.29.5
+ipython==8.26.0
+iterative-telemetry==0.0.8
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+jedi==0.19.1
+jmespath==1.0.1
+joblib==1.4.2
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+jupyter_client==8.6.2
+jupyter_core==5.7.2
+kiwisolver==1.4.5
+kombu==5.4.0
+lark==1.1.9
+lazy_loader==0.4
+librosa==0.10.2.post1
+llamafactory==0.8.4.dev0
+llamafactory==0.8.4.dev0
+llvmlite==0.43.0
+lm-format-enforcer==0.10.3
+lxml==5.3.0
+markdown-it-py==3.0.0
+marshmallow==3.22.0
+matplotlib-inline==0.1.7
+matplotlib==3.9.1
+mdurl==0.1.2
+mmsg==0.1.dev14+gbd2b056.d20240803
+more-itertools==10.3.0
+moviepy==1.0.3
+mpi4py==4.0.0
+mpmath==1.3.0
+msgpack==1.0.8
+multidict==6.0.5
+multiprocess==0.70.16
+mypy-extensions==1.0.0
+nest_asyncio==1.6.0
+networkx==3.3
+ninja==1.11.1.1
+nodeenv==1.9.1
+numba==0.60.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-ml-py==12.535.161
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+nvitop==1.3.2
+omegaconf==2.3.0
+openai==1.37.1
+optree==0.12.1
+ordered-set==4.1.0
+orjson==3.10.6
+outlines==0.0.47.dev37+g26e2934
+packaging==24.1
+packaging==24.1
+pandas==2.2.2
+parso==0.8.4
+pathspec==0.12.1
+pathvalidate==3.0.0
+peft==0.12.0
+pexpect==4.9.0
+pickleshare==0.7.5
+pillow==10.4.0
+pip==24.0
+platformdirs==3.11.0
+platformdirs==4.2.2
+pluggy==1.5.0
+pooch==1.8.2
+pre-commit==3.8.0
+proglog==0.1.10
+prometheus-fastapi-instrumentator==7.0.0
+prometheus_client==0.20.0
+prompt_toolkit==3.0.47
+protobuf==4.25.4
+psutil==6.0.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+py-cpuinfo==9.0.0
+pyairports==2.1.1
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyav==13.0.0
+pycountry==24.6.1
+pycparser==2.22
+pydantic==2.8.2
+pydantic_core==2.20.1
+pydot==3.0.1
+pydub==0.25.1
+pygit2==1.15.1
+pygtrie==2.5.0
+pyparsing==3.1.2
+pytest==8.3.2
+python-dateutil==2.9.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2024.1
+pyzmq==26.0.3
+ranger-fm==1.9.3
+ray==2.34.0
+referencing==0.35.1
+regex==2024.7.24
+requests-toolbelt==1.0.0
+requests==2.32.3
+rfc3986==1.5.0
+rich==13.1.0
+rpds-py==0.19.1
+ruamel.yaml.clib==0.2.8
+ruamel.yaml==0.18.6
+ruff==0.5.5
+s3transfer==0.10.2
+safetensors==0.4.3
+scikit-learn==1.5.1
+scipy==1.14.0
+scmrepo==3.3.7
+semantic-version==2.10.0
+semver==3.0.2
+sentencepiece==0.2.0
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==71.0.4
+shellingham==1.5.4
+shortuuid==1.0.13
+shtab==1.7.1
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+soundfile==0.12.1
+soxr==0.4.0
+sqltrie==0.11.1
+sse-starlette==2.1.3
+stack-data==0.6.2
+starlette==0.37.2
+sympy==1.13.0
+tabulate==0.9.0
+tenacity==8.2.3
+tensorboard-data-server==0.7.2
+tensorboard==2.17.0
+term-image==0.7.2
+termcolor==2.4.0
+threadpoolctl==3.5.0
+tiktoken==0.7.0
+tokenizers==0.19.1
+tomli==2.0.1
+tomlkit==0.12.0
+torch==2.3.1
+torch==2.4.0
+torchaudio==2.4.0
+torchvision==0.18.1
+torchvision==0.19.0
+tornado==6.4.1
+tqdm==4.66.4
+traitlets==5.14.3
+transformers==4.45.0.dev0
+treelib==1.6.4
+triton==2.3.1
+triton==3.0.0
+trl==0.9.6
+typeguard==4.3.0
+typer==0.12.3
+typing-inspect==0.9.0
+typing_extensions==4.12.2
+typing_extensions==4.12.2
+tyro==0.8.10
+tzdata==2024.1
+urllib3==2.2.2
+uvicorn==0.30.3
+uvloop==0.19.0
+vine==5.1.0
+virtualenv==20.26.3
+vllm-flash-attn==2.5.9.post1
+vllm==0.5.3.post1
+voluptuous==0.15.2
+wandb==0.17.5
+watchfiles==0.22.0
+wcwidth==0.2.13
+websockets==11.0.3
+wheel==0.43.0
+wheel==0.43.0
+xformers==0.0.27
+xxhash==3.4.1
+yarl==1.9.4
+zc.lockfile==3.0.post1
+zipp==3.19.2
+zipp==3.19.2
+zstandard==0.23.0

wandb/run-20240922_051551-1efs7bbc/files/wandb-metadata.json

@@ -0,0 +1,1215 @@
+{
+    "os": "Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35",
+    "python": "3.11.9",
+    "heartbeatAt": "2024-09-21T21:15:51.934138",
+    "startedAt": "2024-09-21T21:15:51.132992",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--local_rank=0",
+        "--model_name_or_path",
+        "/aifs4su/yaodong/models/Qwen2-VL-7B-Instruct",
+        "--train_datasets",
+        "/aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted",
+        "--output_dir",
+        "../outputs/dpo_tiv2t_10k_baseline",
+        "--train_data_files",
+        "extracted_preference_10k_washed.json",
+        "--train_template",
+        "NExTQA_preference",
+        "--train_split",
+        "train",
+        "--per_device_train_batch_size",
+        "1",
+        "--per_device_eval_batch_size",
+        "1",
+        "--gradient_accumulation_steps",
+        "1"
+    ],
+    "state": "running",
+    "program": "-m align_anything.trainers.tiv_to_t.dpo",
+    "codePathLocal": null,
+    "git": {
+        "remote": "[email protected]:htlou/align-anything.git",
+        "commit": "9eff99b0ee34f7265b8a2e1824650634c898f654"
+    },
+    "email": "[email protected]",
+    "root": "/aifs4su/yaodong/projects/hantao/dev_cham/align-anything",
+    "host": "dgx-081",
+    "username": "yangyaodong",
+    "executable": "/aifs4su/yaodong/miniconda3/envs/hantao_tiv/bin/python3.11",
+    "cpu_count": 112,
+    "cpu_count_logical": 224,
+    "cpu_freq": {
+        "current": 2100.4980491071433,
+        "min": 800.0,
+        "max": 3800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3792.33,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.003,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.996,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.003,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.003,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.003,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 1758.8549690246582,
+            "used": 58.09828567504883
+        }
+    },
+    "gpu": "NVIDIA H800",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        }
+    ],
+    "memory": {
+        "total": 2015.5615730285645
+    }
+}

wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/step": 75, "train/loss": 2.442300796508789, "train/reward": -2.549769401550293, "train/better_sample_reward": -1.6329433917999268, "train/worse_sample_reward": -0.9168255925178528, "train/reward_accuracy": 0.6875, "train/reward_margin": -0.716117799282074, "train/lr": 9.975195304536758e-07, "train/epoch": 0.12396694214876033, "_timestamp": 1726954955.677539, "_runtime": 1604.5298900604248, "_step": 75}

wandb/run-20240922_051551-1efs7bbc/logs/debug.log

@@ -0,0 +1,29 @@
+2024-09-22 05:15:51,141 INFO    MainThread:1226843 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-09-22 05:15:51,141 INFO    MainThread:1226843 [wandb_setup.py:_flush():76] Configure stats pid to 1226843
+2024-09-22 05:15:51,141 INFO    MainThread:1226843 [wandb_setup.py:_flush():76] Loading settings from /home/yangyaodong/.config/wandb/settings
+2024-09-22 05:15:51,141 INFO    MainThread:1226843 [wandb_setup.py:_flush():76] Loading settings from /aifs4su/yaodong/projects/hantao/dev_cham/align-anything/scripts/wandb/settings
+2024-09-22 05:15:51,141 INFO    MainThread:1226843 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***'}
+2024-09-22 05:15:51,141 INFO    MainThread:1226843 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-09-22 05:15:51,141 WARNING MainThread:1226843 [wandb_setup.py:_flush():76] Could not find program at -m align_anything.trainers.tiv_to_t.dpo
+2024-09-22 05:15:51,141 INFO    MainThread:1226843 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.tiv_to_t.dpo'}
+2024-09-22 05:15:51,141 INFO    MainThread:1226843 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-09-22 05:15:51,141 INFO    MainThread:1226843 [wandb_init.py:_log_setup():529] Logging user logs to ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/logs/debug.log
+2024-09-22 05:15:51,142 INFO    MainThread:1226843 [wandb_init.py:_log_setup():530] Logging internal logs to ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/logs/debug-internal.log
+2024-09-22 05:15:51,142 INFO    MainThread:1226843 [wandb_init.py:init():569] calling init triggers
+2024-09-22 05:15:51,142 INFO    MainThread:1226843 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1.0, 'per_device_eval_batch_size': 1.0, 'gradient_accumulation_steps': 1.0, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.01, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'regularization': 0.001, 'scale_coeff': 0.1, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False}, 'data_cfgs': {'train_datasets': '/aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted', 'train_template': 'NExTQA_preference', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'extracted_preference_10k_washed.json', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'dpo', 'output_dir': '../outputs/dpo_tiv2t_10k_baseline', 'cache_dir': None, 'save_interval': 100000}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/models/Qwen2-VL-7B-Instruct', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
+2024-09-22 05:15:51,142 INFO    MainThread:1226843 [wandb_init.py:init():619] starting backend
+2024-09-22 05:15:51,142 INFO    MainThread:1226843 [wandb_init.py:init():623] setting up manager
+2024-09-22 05:15:51,143 INFO    MainThread:1226843 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-09-22 05:15:51,147 INFO    MainThread:1226843 [wandb_init.py:init():631] backend started and connected
+2024-09-22 05:15:51,150 INFO    MainThread:1226843 [wandb_init.py:init():720] updated telemetry
+2024-09-22 05:15:51,170 INFO    MainThread:1226843 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-09-22 05:15:51,600 INFO    MainThread:1226843 [wandb_run.py:_on_init():2435] communicating current version
+2024-09-22 05:15:51,779 INFO    MainThread:1226843 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.18.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-09-22 05:15:51,779 INFO    MainThread:1226843 [wandb_init.py:init():804] starting run threads in backend
+2024-09-22 05:15:57,167 INFO    MainThread:1226843 [wandb_run.py:_console_start():2413] atexit reg
+2024-09-22 05:15:57,167 INFO    MainThread:1226843 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-09-22 05:15:57,167 INFO    MainThread:1226843 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-09-22 05:15:57,167 INFO    MainThread:1226843 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-09-22 05:15:57,170 INFO    MainThread:1226843 [wandb_init.py:init():847] run started, returning control to user process

wandb/run-20240922_063327-rptdqsqq/files/conda-environment.yaml

@@ -0,0 +1,504 @@
+name: hantao_new
+channels:
+  - nvidia/label/cuda-12.1.0
+  - nvidia
+  - pytorch
+  - huggingface
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - aom=3.9.1=hac33072_0
+  - asttokens=2.4.1=pyhd8ed1ab_0
+  - blas=1.0=mkl
+  - brotli-python=1.1.0=py311hb755f60_1
+  - bzip2=1.0.8=h4bc722e_7
+  - ca-certificates=2024.7.4=hbcca054_0
+  - cairo=1.18.0=hebfffa5_3
+  - certifi=2024.7.4=pyhd8ed1ab_0
+  - cffi=1.16.0=py311hb3a22ac_0
+  - charset-normalizer=3.3.2=pyhd8ed1ab_0
+  - comm=0.2.2=pyhd8ed1ab_0
+  - cuda=12.1.0=0
+  - cuda-cccl=12.1.55=0
+  - cuda-command-line-tools=12.1.0=0
+  - cuda-compiler=12.1.0=0
+  - cuda-cudart=12.1.55=0
+  - cuda-cudart-dev=12.1.55=0
+  - cuda-cudart-static=12.1.55=0
+  - cuda-cuobjdump=12.1.55=0
+  - cuda-cupti=12.1.62=0
+  - cuda-cupti-static=12.1.62=0
+  - cuda-cuxxfilt=12.1.55=0
+  - cuda-demo-suite=12.1.55=0
+  - cuda-documentation=12.1.55=0
+  - cuda-driver-dev=12.1.55=0
+  - cuda-gdb=12.1.55=0
+  - cuda-libraries=12.1.0=0
+  - cuda-libraries-dev=12.1.0=0
+  - cuda-libraries-static=12.1.0=0
+  - cuda-nsight=12.1.55=0
+  - cuda-nsight-compute=12.1.0=0
+  - cuda-nvcc=12.1.66=0
+  - cuda-nvdisasm=12.1.55=0
+  - cuda-nvml-dev=12.1.55=0
+  - cuda-nvprof=12.1.55=0
+  - cuda-nvprune=12.1.55=0
+  - cuda-nvrtc=12.1.55=0
+  - cuda-nvrtc-dev=12.1.55=0
+  - cuda-nvrtc-static=12.1.55=0
+  - cuda-nvtx=12.1.66=0
+  - cuda-nvvp=12.1.55=0
+  - cuda-opencl=12.1.56=0
+  - cuda-opencl-dev=12.1.56=0
+  - cuda-profiler-api=12.1.55=0
+  - cuda-runtime=12.1.0=0
+  - cuda-sanitizer-api=12.1.55=0
+  - cuda-toolkit=12.1.0=0
+  - cuda-tools=12.1.0=0
+  - cuda-visual-tools=12.1.0=0
+  - dav1d=1.2.1=hd590300_0
+  - debugpy=1.8.2=py311h4332511_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - exceptiongroup=1.2.2=pyhd8ed1ab_0
+  - executing=2.0.1=pyhd8ed1ab_0
+  - expat=2.6.2=h59595ed_0
+  - ffmpeg=7.0.1=gpl_h9be9148_104
+  - filelock=3.15.4=pyhd8ed1ab_0
+  - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
+  - font-ttf-inconsolata=3.000=h77eed37_0
+  - font-ttf-source-code-pro=2.038=h77eed37_0
+  - font-ttf-ubuntu=0.83=h77eed37_2
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-conda-ecosystem=1=0
+  - fonts-conda-forge=1=0
+  - freetype=2.12.1=h267a509_2
+  - fribidi=1.0.10=h36c2ea0_0
+  - gds-tools=1.6.0.25=0
+  - gettext=0.22.5=h59595ed_2
+  - gettext-tools=0.22.5=h59595ed_2
+  - git-lfs=3.5.1=ha770c72_0
+  - gmp=6.3.0=hac33072_2
+  - gmpy2=2.1.5=py311hc4f1f91_1
+  - gnutls=3.7.9=hb077bed_0
+  - graphite2=1.3.13=h59595ed_1003
+  - h2=4.1.0=pyhd8ed1ab_0
+  - harfbuzz=9.0.0=hda332d3_1
+  - hpack=4.0.0=pyh9f0ad1d_0
+  - hyperframe=6.0.1=pyhd8ed1ab_0
+  - icu=75.1=he02047a_0
+  - idna=3.7=pyhd8ed1ab_0
+  - importlib-metadata=8.2.0=pyha770c72_0
+  - importlib_metadata=8.2.0=hd8ed1ab_0
+  - intel-openmp=2022.0.1=h06a4308_3633
+  - ipykernel=6.29.5=pyh3099207_0
+  - ipython=8.26.0=pyh707e725_0
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jinja2=3.1.4=pyhd8ed1ab_0
+  - jupyter_client=8.6.2=pyhd8ed1ab_0
+  - jupyter_core=5.7.2=py311h38be061_0
+  - keyutils=1.6.1=h166bdaf_0
+  - krb5=1.21.3=h659f571_0
+  - lame=3.100=h166bdaf_1003
+  - lcms2=2.16=hb7c19ff_0
+  - ld_impl_linux-64=2.40=hf3520f5_7
+  - lerc=4.0.0=h27087fc_0
+  - libabseil=20240116.2=cxx17_he02047a_1
+  - libasprintf=0.22.5=h661eb56_2
+  - libasprintf-devel=0.22.5=h661eb56_2
+  - libass=0.17.1=h39113c1_2
+  - libblas=3.9.0=16_linux64_mkl
+  - libcblas=3.9.0=16_linux64_mkl
+  - libcublas=12.1.0.26=0
+  - libcublas-dev=12.1.0.26=0
+  - libcublas-static=12.1.0.26=0
+  - libcufft=11.0.2.4=0
+  - libcufft-dev=11.0.2.4=0
+  - libcufft-static=11.0.2.4=0
+  - libcufile=1.6.0.25=0
+  - libcufile-dev=1.6.0.25=0
+  - libcufile-static=1.6.0.25=0
+  - libcurand=10.3.2.56=0
+  - libcurand-dev=10.3.2.56=0
+  - libcurand-static=10.3.2.56=0
+  - libcusolver=11.4.4.55=0
+  - libcusolver-dev=11.4.4.55=0
+  - libcusolver-static=11.4.4.55=0
+  - libcusparse=12.0.2.55=0
+  - libcusparse-dev=12.0.2.55=0
+  - libcusparse-static=12.0.2.55=0
+  - libdeflate=1.20=hd590300_0
+  - libdrm=2.4.122=h4ab18f5_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libexpat=2.6.2=h59595ed_0
+  - libffi=3.4.2=h7f98852_5
+  - libgcc-ng=14.1.0=h77fa898_0
+  - libgettextpo=0.22.5=h59595ed_2
+  - libgettextpo-devel=0.22.5=h59595ed_2
+  - libglib=2.80.3=h8a4344b_1
+  - libgomp=14.1.0=h77fa898_0
+  - libhwloc=2.11.1=default_hecaa2ac_1000
+  - libiconv=1.17=hd590300_2
+  - libidn2=2.3.7=hd590300_0
+  - libjpeg-turbo=3.0.0=hd590300_1
+  - liblapack=3.9.0=16_linux64_mkl
+  - libnpp=12.0.2.50=0
+  - libnpp-dev=12.0.2.50=0
+  - libnpp-static=12.0.2.50=0
+  - libnsl=2.0.1=hd590300_0
+  - libnvjitlink=12.1.105=0
+  - libnvjitlink-dev=12.1.55=0
+  - libnvjpeg=12.1.0.39=0
+  - libnvjpeg-dev=12.1.0.39=0
+  - libnvjpeg-static=12.1.0.39=0
+  - libnvvm-samples=12.1.55=0
+  - libopenvino=2024.2.0=h2da1b83_1
+  - libopenvino-auto-batch-plugin=2024.2.0=hb045406_1
+  - libopenvino-auto-plugin=2024.2.0=hb045406_1
+  - libopenvino-hetero-plugin=2024.2.0=h5c03a75_1
+  - libopenvino-intel-cpu-plugin=2024.2.0=h2da1b83_1
+  - libopenvino-intel-gpu-plugin=2024.2.0=h2da1b83_1
+  - libopenvino-intel-npu-plugin=2024.2.0=he02047a_1
+  - libopenvino-ir-frontend=2024.2.0=h5c03a75_1
+  - libopenvino-onnx-frontend=2024.2.0=h07e8aee_1
+  - libopenvino-paddle-frontend=2024.2.0=h07e8aee_1
+  - libopenvino-pytorch-frontend=2024.2.0=he02047a_1
+  - libopenvino-tensorflow-frontend=2024.2.0=h39126c6_1
+  - libopenvino-tensorflow-lite-frontend=2024.2.0=he02047a_1
+  - libopus=1.3.1=h7f98852_1
+  - libpciaccess=0.18=hd590300_0
+  - libpng=1.6.43=h2797004_0
+  - libprotobuf=4.25.3=h08a7969_0
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsqlite=3.46.0=hde9e2c9_0
+  - libstdcxx-ng=14.1.0=hc0a3c3a_0
+  - libtasn1=4.19.0=h166bdaf_0
+  - libtiff=4.6.0=h1dd3fc0_3
+  - libunistring=0.9.10=h7f98852_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libva=2.22.0=hb711507_0
+  - libvpx=1.14.1=hac33072_0
+  - libwebp-base=1.4.0=hd590300_0
+  - libxcb=1.16=hd590300_0
+  - libxcrypt=4.4.36=hd590300_1
+  - libxml2=2.12.7=he7c6b58_4
+  - libzlib=1.3.1=h4ab18f5_1
+  - llvm-openmp=15.0.7=h0cdce71_0
+  - markupsafe=2.1.5=py311h459d7ec_0
+  - matplotlib-inline=0.1.7=pyhd8ed1ab_0
+  - mkl=2022.1.0=hc2b9512_224
+  - mpc=1.3.1=hfe3b2da_0
+  - mpfr=4.2.1=h38ae2d0_2
+  - mpmath=1.3.0=pyhd8ed1ab_0
+  - ncurses=6.5=h59595ed_0
+  - nest-asyncio=1.6.0=pyhd8ed1ab_0
+  - nettle=3.9.1=h7ab15ed_0
+  - networkx=3.3=pyhd8ed1ab_1
+  - nsight-compute=2023.1.0.15=0
+  - ocl-icd=2.3.2=hd590300_1
+  - openh264=2.4.1=h59595ed_0
+  - openjpeg=2.5.2=h488ebb8_0
+  - openssl=3.3.1=h4bc722e_2
+  - p11-kit=0.24.1=hc5aa10d_0
+  - packaging=24.1=pyhd8ed1ab_0
+  - parso=0.8.4=pyhd8ed1ab_0
+  - pcre2=10.44=h0f59acf_0
+  - pexpect=4.9.0=pyhd8ed1ab_0
+  - pickleshare=0.7.5=py_1003
+  - pillow=10.4.0=py311h82a398c_0
+  - pip=24.0=pyhd8ed1ab_0
+  - pixman=0.43.2=h59595ed_0
+  - prompt-toolkit=3.0.47=pyha770c72_0
+  - psutil=6.0.0=py311h331c9d8_0
+  - pthread-stubs=0.4=h36c2ea0_1001
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pugixml=1.14=h59595ed_0
+  - pure_eval=0.2.3=pyhd8ed1ab_0
+  - pycparser=2.22=pyhd8ed1ab_0
+  - pygments=2.18.0=pyhd8ed1ab_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - python=3.11.9=hb806964_0_cpython
+  - python_abi=3.11=4_cp311
+  - pytorch=2.4.0=py3.11_cuda12.1_cudnn9.1.0_0
+  - pytorch-cuda=12.1=ha16c6d3_5
+  - pytorch-mutex=1.0=cuda
+  - pyyaml=6.0.1=py311h459d7ec_1
+  - pyzmq=26.0.3=py311h08a0b41_0
+  - readline=8.2=h8228510_1
+  - requests=2.32.3=pyhd8ed1ab_0
+  - setuptools=71.0.4=pyhd8ed1ab_0
+  - six=1.16.0=pyh6c4a22f_0
+  - snappy=1.2.1=ha2e4443_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - svt-av1=2.1.2=hac33072_0
+  - sympy=1.13.0=pypyh2585a3b_103
+  - tbb=2021.12.0=h434a139_3
+  - tk=8.6.13=noxft_h4845f30_101
+  - torchaudio=2.4.0=py311_cu121
+  - torchtriton=3.0.0=py311
+  - tornado=6.4.1=py311h331c9d8_0
+  - traitlets=5.14.3=pyhd8ed1ab_0
+  - typing_extensions=4.12.2=pyha770c72_0
+  - urllib3=2.2.2=pyhd8ed1ab_1
+  - wayland=1.23.0=h5291e77_0
+  - wayland-protocols=1.36=hd8ed1ab_0
+  - wcwidth=0.2.13=pyhd8ed1ab_0
+  - wheel=0.43.0=pyhd8ed1ab_1
+  - x264=1!164.3095=h166bdaf_2
+  - x265=3.5=h924138e_3
+  - xorg-fixesproto=5.0=h7f98852_1002
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.9=hb711507_1
+  - xorg-libxau=1.0.11=hd590300_0
+  - xorg-libxdmcp=1.1.3=h7f98852_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxfixes=5.0.3=h7f98852_1004
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xproto=7.0.31=h7f98852_1007
+  - xz=5.2.6=h166bdaf_0
+  - yaml=0.2.5=h7f98852_2
+  - zeromq=4.3.5=h75354e8_4
+  - zipp=3.19.2=pyhd8ed1ab_0
+  - zlib=1.3.1=h4ab18f5_1
+  - zstandard=0.23.0=py311h5cd10c7_0
+  - zstd=1.5.6=ha6fb4c9_0
+  - pip:
+      - about-time==4.2.1
+      - absl-py==2.1.0
+      - accelerate==0.33.0
+      - aiofiles==23.2.1
+      - aiohttp==3.9.5
+      - aiohttp-retry==2.8.3
+      - aiosignal==1.3.1
+      - align-anything==0.0.1.dev0
+      - alive-progress==3.1.5
+      - amqp==5.2.0
+      - annotated-types==0.7.0
+      - antlr4-python3-runtime==4.9.3
+      - anyio==4.4.0
+      - appdirs==1.4.4
+      - asyncssh==2.16.0
+      - atpublic==5.0
+      - attrs==23.2.0
+      - audioread==3.0.1
+      - av==13.0.0
+      - backoff==2.2.1
+      - billiard==4.2.0
+      - boto3==1.35.9
+      - botocore==1.35.9
+      - cachetools==5.4.0
+      - celery==5.4.0
+      - cfgv==3.4.0
+      - click==8.1.7
+      - click-didyoumean==0.3.1
+      - click-plugins==1.1.1
+      - click-repl==0.3.0
+      - cloudpickle==3.0.0
+      - cmake==3.30.1
+      - colorama==0.4.6
+      - commonmark==0.9.1
+      - configobj==5.0.8
+      - contourpy==1.2.1
+      - cryptography==43.0.0
+      - cycler==0.12.1
+      - dacite==1.6.0
+      - dagshub==0.3.35
+      - dagshub-annotation-converter==0.1.0
+      - dataclasses-json==0.6.7
+      - datasets==2.20.0
+      - deepspeed==0.14.4
+      - dictdiffer==0.9.0
+      - diffusers==0.29.2
+      - dill==0.3.8
+      - diskcache==5.6.3
+      - distlib==0.3.8
+      - distro==1.9.0
+      - dnspython==2.6.1
+      - docker-pycreds==0.4.0
+      - dpath==2.2.0
+      - dulwich==0.22.1
+      - dvc==3.55.1
+      - dvc-data==3.16.5
+      - dvc-http==2.32.0
+      - dvc-objects==5.1.0
+      - dvc-render==1.0.2
+      - dvc-studio-client==0.21.0
+      - dvc-task==0.4.0
+      - email-validator==2.2.0
+      - entrypoints==0.4
+      - fastapi==0.111.1
+      - fastapi-cli==0.0.4
+      - ffmpy==0.3.3
+      - flatten-dict==0.4.2
+      - flufl-lock==8.1.0
+      - fonttools==4.53.1
+      - frozenlist==1.4.1
+      - fsspec==2024.5.0
+      - funcy==2.0
+      - fusepy==3.0.1
+      - gitdb==4.0.11
+      - gitpython==3.1.43
+      - gql==3.5.0
+      - gradio==4.39.0
+      - gradio-client==1.1.1
+      - gradio-multimodalchatbot==0.0.1
+      - grandalf==0.8
+      - grapheme==0.6.0
+      - graphql-core==3.2.3
+      - grpcio==1.65.1
+      - gto==1.7.1
+      - h11==0.14.0
+      - hjson==3.1.0
+      - httpcore==0.16.3
+      - httptools==0.6.1
+      - httpx==0.23.3
+      - huggingface-hub==0.24.3
+      - hydra-core==1.3.2
+      - identify==2.6.0
+      - importlib-resources==6.4.0
+      - iniconfig==2.0.0
+      - interegular==0.3.3
+      - iterative-telemetry==0.0.8
+      - jmespath==1.0.1
+      - joblib==1.4.2
+      - jsonschema==4.23.0
+      - jsonschema-specifications==2023.12.1
+      - kiwisolver==1.4.5
+      - kombu==5.4.0
+      - lark==1.1.9
+      - lazy-loader==0.4
+      - librosa==0.10.2.post1
+      - llvmlite==0.43.0
+      - lm-format-enforcer==0.10.3
+      - lxml==5.3.0
+      - markdown==3.6
+      - markdown-it-py==3.0.0
+      - marshmallow==3.22.0
+      - matplotlib==3.9.1
+      - mdurl==0.1.2
+      - mmsg==0.1.dev14+gbd2b056.d20240803
+      - msgpack==1.0.8
+      - multidict==6.0.5
+      - multiprocess==0.70.16
+      - mypy-extensions==1.0.0
+      - ninja==1.11.1.1
+      - nodeenv==1.9.1
+      - numba==0.60.0
+      - numpy==1.26.4
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu12==8.9.2.26
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-ml-py==12.535.161
+      - nvidia-nccl-cu12==2.20.5
+      - nvidia-nvjitlink-cu12==12.5.82
+      - nvidia-nvtx-cu12==12.1.105
+      - nvitop==1.3.2
+      - omegaconf==2.3.0
+      - openai==1.37.1
+      - optree==0.12.1
+      - orjson==3.10.6
+      - outlines==0.0.47.dev37+g26e2934
+      - pandas==2.2.2
+      - pathspec==0.12.1
+      - pathvalidate==3.0.0
+      - peft==0.12.0
+      - platformdirs==3.11.0
+      - pluggy==1.5.0
+      - pooch==1.8.2
+      - pre-commit==3.8.0
+      - prometheus-client==0.20.0
+      - prometheus-fastapi-instrumentator==7.0.0
+      - protobuf==4.25.4
+      - py-cpuinfo==9.0.0
+      - pyairports==2.1.1
+      - pyarrow==17.0.0
+      - pyarrow-hotfix==0.6
+      - pycountry==24.6.1
+      - pydantic==2.8.2
+      - pydantic-core==2.20.1
+      - pydot==3.0.1
+      - pydub==0.25.1
+      - pygit2==1.15.1
+      - pygtrie==2.5.0
+      - pyparsing==3.1.2
+      - pytest==8.3.2
+      - python-dateutil==2.9.0.post0
+      - python-dotenv==1.0.1
+      - python-multipart==0.0.9
+      - pytz==2024.1
+      - ranger-fm==1.9.3
+      - ray==2.34.0
+      - referencing==0.35.1
+      - regex==2024.7.24
+      - requests-toolbelt==1.0.0
+      - rfc3986==1.5.0
+      - rich==13.1.0
+      - rpds-py==0.19.1
+      - ruamel-yaml==0.18.6
+      - ruamel-yaml-clib==0.2.8
+      - ruff==0.5.5
+      - s3transfer==0.10.2
+      - safetensors==0.4.3
+      - scikit-learn==1.5.1
+      - scipy==1.14.0
+      - scmrepo==3.3.7
+      - semantic-version==2.10.0
+      - semver==3.0.2
+      - sentencepiece==0.2.0
+      - sentry-sdk==2.11.0
+      - setproctitle==1.3.3
+      - shellingham==1.5.4
+      - shortuuid==1.0.13
+      - shtab==1.7.1
+      - smmap==5.0.1
+      - sniffio==1.3.1
+      - soundfile==0.12.1
+      - soxr==0.4.0
+      - sqltrie==0.11.1
+      - starlette==0.37.2
+      - tabulate==0.9.0
+      - tenacity==8.2.3
+      - tensorboard==2.17.0
+      - tensorboard-data-server==0.7.2
+      - term-image==0.7.2
+      - termcolor==2.4.0
+      - threadpoolctl==3.5.0
+      - tiktoken==0.7.0
+      - tokenizers==0.19.1
+      - tomlkit==0.12.0
+      - torch==2.3.1
+      - torchvision==0.18.1
+      - tqdm==4.66.4
+      - transformers==4.44.0.dev0
+      - treelib==1.6.4
+      - triton==2.3.1
+      - typer==0.12.3
+      - typing-inspect==0.9.0
+      - tzdata==2024.1
+      - uvicorn==0.30.3
+      - uvloop==0.19.0
+      - vine==5.1.0
+      - virtualenv==20.26.3
+      - vllm==0.5.3.post1
+      - vllm-flash-attn==2.5.9.post1
+      - voluptuous==0.15.2
+      - wandb==0.17.5
+      - watchfiles==0.22.0
+      - websockets==11.0.3
+      - werkzeug==3.0.3
+      - xformers==0.0.27
+      - xxhash==3.4.1
+      - yarl==1.9.4
+      - zc-lockfile==3.0.post1
+prefix: /aifs4su/yaodong/miniconda3/envs/hantao_new

wandb/run-20240922_063327-rptdqsqq/files/config.yaml

@@ -0,0 +1,112 @@
+wandb_version: 1
+
+train_cfgs:
+  desc: null
+  value:
+    ds_cfgs: ds_z3_config.json
+    epochs: 3
+    seed: 42
+    per_device_train_batch_size: 1.0
+    per_device_eval_batch_size: 1.0
+    gradient_accumulation_steps: 1.0
+    gradient_checkpointing: true
+    learning_rate: 1.0e-06
+    lr_scheduler_type: cosine
+    lr_warmup_ratio: 0.01
+    weight_decay: 0.0
+    adam_betas:
+    - 0.9
+    - 0.95
+    bf16: true
+    fp16: false
+    eval_strategy: epoch
+    eval_interval: 10
+    regularization: 0.001
+    scale_coeff: 0.1
+    freeze_mm_proj: false
+    freeze_vision_tower: true
+    freeze_language_model: false
+data_cfgs:
+  desc: null
+  value:
+    train_datasets: /aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted
+    train_template: NExTQA_preference
+    train_size: null
+    train_split: train
+    train_subset: null
+    train_data_files: extracted_preference_10k_washed.json
+    train_optional_args: []
+    eval_datasets: null
+    eval_template: null
+    eval_size: null
+    eval_split: null
+    eval_subset: null
+    eval_data_files: null
+    eval_optional_args: []
+logger_cfgs:
+  desc: null
+  value:
+    log_type: wandb
+    log_project: align-anything
+    log_run_name: dpo
+    output_dir: ../outputs/dpo_tiv2t_10k_baseline
+    cache_dir: null
+    save_interval: 100000
+model_cfgs:
+  desc: null
+  value:
+    model_name_or_path: /aifs4su/yaodong/models/Qwen2-VL-7B-Instruct
+    trust_remote_code: true
+    model_max_length: 4096
+special_tokens:
+  desc: null
+  value: null
+_wandb:
+  desc: null
+  value:
+    python_version: 3.11.9
+    cli_version: 0.17.5
+    framework: huggingface
+    huggingface_version: 4.45.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1726958007
+    t:
+      1:
+      - 1
+      - 5
+      - 11
+      - 41
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 83
+      - 98
+      - 105
+      2:
+      - 1
+      - 5
+      - 11
+      - 41
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 83
+      - 98
+      - 105
+      3:
+      - 2
+      - 13
+      - 16
+      - 23
+      - 61
+      4: 3.11.9
+      5: 0.17.5
+      6: 4.45.0.dev0
+      8:
+      - 5
+      13: linux-x86_64

wandb/run-20240922_063327-rptdqsqq/files/output.log

@@ -0,0 +1,2038 @@
+
+***** Running training *****
+
+Training 1/3 epoch:   0%|          | 0/1812 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...
+/aifs4su/yaodong/miniconda3/envs/hantao_tiv/lib/python3.11/site-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
+  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.5581):   0%|          | 9/1812 [02:28<7:23:49, 14.77s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.8237):   1%|          | 19/1812 [05:06<7:47:17, 15.64s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.6081):   2%|▏         | 29/1812 [09:07<13:20:02, 26.92s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 4.5173):   2%|▏         | 39/1812 [12:22<8:51:23, 17.98s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.0933):   3%|▎         | 49/1812 [15:31<9:27:49, 19.32s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.5557):   3%|▎         | 59/1812 [18:52<10:11:06, 20.92s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 2.5379):   4%|▍         | 69/1812 [23:01<11:26:14, 23.62s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.1026):   4%|▍         | 79/1812 [26:23<9:44:28, 20.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 2.3413):   5%|▍         | 90/1812 [30:20<9:17:33, 19.43s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.6764):   6%|▌         | 100/1812 [34:08<10:18:43, 21.68s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.8155):   6%|▌         | 110/1812 [36:52<6:49:29, 14.44s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.6338):   7%|▋         | 120/1812 [40:19<9:46:49, 20.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.9787):   7%|▋         | 130/1812 [43:11<7:43:29, 16.53s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.6081):   8%|▊         | 140/1812 [46:35<10:47:06, 23.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 2.1454):   8%|▊         | 150/1812 [49:39<8:26:05, 18.27s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.8813):   9%|▉         | 160/1812 [53:08<9:32:15, 20.78s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.7650):   9%|▉         | 170/1812 [56:23<9:40:36, 21.22s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.5732):  10%|▉         | 180/1812 [59:44<9:28:12, 20.89s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.3015):  10%|█         | 190/1812 [1:02:33<6:35:30, 14.63s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.9606):  11%|█         | 200/1812 [1:05:46<8:47:32, 19.64s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.2718):  12%|█▏        | 210/1812 [1:09:17<9:09:28, 20.58s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.6807):  12%|█▏        | 220/1812 [1:12:20<8:15:40, 18.68s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 2.6464):  13%|█▎        | 230/1812 [1:15:30<8:06:16, 18.44s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.7252):  13%|█▎        | 239/1812 [1:17:56<7:44:51, 17.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.3659):  14%|█▍        | 250/1812 [1:22:10<9:17:00, 21.40s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 2.0745):  14%|█▍        | 260/1812 [1:25:36<7:44:56, 17.97s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.7852):  15%|█▍        | 270/1812 [1:29:53<10:23:13, 24.25s/it]
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.8651):  15%|█▌        | 279/1812 [1:32:42<6:45:04, 15.85s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.7337):  16%|█▌        | 290/1812 [1:36:15<7:06:11, 16.80s/it]
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.4919):  17%|█▋        | 299/1812 [1:39:10<8:29:29, 20.20s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.0384):  17%|█▋        | 310/1812 [1:42:13<6:59:35, 16.76s/it]
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 2.0364):  18%|█▊        | 319/1812 [1:44:31<5:51:07, 14.11s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.0456):  18%|█▊        | 329/1812 [1:47:41<7:11:57, 17.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.4968):  19%|█▉        | 340/1812 [1:50:58<9:39:37, 23.63s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.1733):  19%|█▉        | 350/1812 [1:54:26<9:52:07, 24.30s/it]
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.8559):  20%|█▉        | 359/1812 [1:57:35<8:58:23, 22.23s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.6638):  20%|██        | 369/1812 [2:00:38<7:18:02, 18.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.0443):  21%|██        | 380/1812 [2:04:26<8:24:06, 21.12s/it]
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.9396):  21%|██▏       | 389/1812 [2:06:51<7:01:29, 17.77s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.9349):  22%|██▏       | 400/1812 [2:10:52<10:02:17, 25.59s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.7493):  23%|██���       | 410/1812 [2:14:50<9:18:12, 23.89s/it]
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.7967):  23%|██▎       | 419/1812 [2:17:11<5:30:25, 14.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.2191):  24%|██▎       | 430/1812 [2:21:13<7:47:05, 20.28s/it]
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.6274):  24%|██▍       | 439/1812 [2:24:24<7:05:58, 18.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.5032):  25%|██▍       | 450/1812 [2:27:53<7:25:34, 19.63s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.7499):  25%|██▌       | 460/1812 [2:31:20<7:49:47, 20.85s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.6526):  26%|██▌       | 470/1812 [2:34:52<7:52:39, 21.13s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.7733):  26%|██▋       | 480/1812 [2:38:19<9:04:34, 24.53s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.7606):  27%|██▋       | 490/1812 [2:41:25<7:01:16, 19.12s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.2104):  28%|██▊       | 500/1812 [2:44:07<6:35:38, 18.09s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.6659):  28%|██▊       | 510/1812 [2:48:05<8:19:57, 23.04s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 2.7401):  29%|██▊       | 520/1812 [2:51:33<6:22:57, 17.78s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.1348):  29%|██▉       | 530/1812 [2:54:38<6:23:21, 17.94s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.3653):  30%|██▉       | 540/1812 [2:58:00<7:06:12, 20.10s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 2.8389):  30%|███       | 550/1812 [3:01:04<6:42:10, 19.12s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 1.1369):  31%|███       | 560/1812 [3:03:55<5:27:10, 15.68s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 2.6080):  31%|███▏      | 570/1812 [3:07:28<9:08:24, 26.49s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.6089):  32%|███▏      | 580/1812 [3:10:54<8:43:20, 25.49s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 0.6223):  33%|███▎      | 590/1812 [3:13:43<5:30:33, 16.23s/it]
+
+
+
+
+
+
+
+
+
+
+Training 1/3 epoch (loss 2.0839):  33%|███▎      | 600/1812 [3:17:13<6:15:47, 18.60s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.7982):  34%|███▎      | 610/1812 [3:20:10<5:39:36, 16.95s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 1.2234):  34%|███▍      | 620/1812 [3:22:42<5:10:04, 15.61s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.8536):  35%|███▍      | 630/1812 [3:25:55<7:27:42, 22.73s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.3619):  35%|███▌      | 640/1812 [3:29:48<6:55:50, 21.29s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 1.5365):  36%|███▌      | 650/1812 [3:32:50<6:27:08, 19.99s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.8612):  36%|███▋      | 660/1812 [3:35:58<6:15:08, 19.54s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.9075):  37%|███▋      | 670/1812 [3:39:55<6:51:45, 21.63s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.4571):  38%|███▊      | 680/1812 [3:43:27<6:14:56, 19.87s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.1629):  38%|███▊      | 690/1812 [3:47:13<7:27:32, 23.93s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.2249):  39%|███▊      | 700/1812 [3:50:57<7:42:23, 24.95s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.5625):  39%|███▉      | 710/1812 [3:54:03<4:58:24, 16.25s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.8886):  40%|███▉      | 720/1812 [3:56:56<6:19:30, 20.85s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.9067):  40%|████      | 730/1812 [4:00:23<6:20:53, 21.12s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.4235):  41%|████      | 740/1812 [4:03:17<6:26:20, 21.62s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 1.5445):  41%|████▏     | 750/1812 [4:06:34<6:00:27, 20.36s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 1.0290):  42%|████▏     | 760/1812 [4:09:39<5:16:35, 18.06s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.1481):  42%|████▏     | 770/1812 [4:13:06<5:44:37, 19.84s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 1.0937):  43%|████▎     | 780/1812 [4:16:35<6:09:28, 21.48s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.6625):  44%|████▎     | 790/1812 [4:19:40<5:18:15, 18.68s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.3620):  44%|████▍     | 800/1812 [4:22:28<4:26:48, 15.82s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 2.0399):  45%|████▍     | 810/1812 [4:26:12<6:33:18, 23.55s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.3767):  45%|████▌     | 820/1812 [4:29:11<4:46:45, 17.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.2618):  46%|████▌     | 830/1812 [4:32:23<5:13:18, 19.14s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.4501):  46%|████▋     | 840/1812 [4:35:20<5:35:26, 20.71s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.5268):  47%|████▋     | 850/1812 [4:38:52<6:10:50, 23.13s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 1.5898):  47%|████▋     | 860/1812 [4:42:39<5:40:40, 21.47s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.1709):  48%|████▊     | 870/1812 [4:46:22<6:16:43, 23.99s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.1749):  49%|████▊     | 880/1812 [4:50:09<5:00:35, 19.35s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 1.1029):  49%|████▉     | 890/1812 [4:53:16<5:21:44, 20.94s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.6366):  50%|████▉     | 900/1812 [4:56:17<4:28:31, 17.67s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.9695):  50%|█████     | 910/1812 [4:59:17<4:19:56, 17.29s/it]
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.2622):  51%|█████     | 919/1812 [5:01:48<4:21:28, 17.57s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.9309):  51%|█████▏    | 930/1812 [5:04:59<5:03:24, 20.64s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 1.0294):  52%|█████▏    | 940/1812 [5:07:41<4:13:34, 17.45s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.3830):  52%|█████▏    | 950/1812 [5:11:03<4:22:12, 18.25s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.4278):  53%|█████▎    | 960/1812 [5:14:36<5:08:56, 21.76s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.1940):  54%|█████▎    | 970/1812 [5:17:58<5:11:47, 22.22s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.0402):  54%|█████▍    | 980/1812 [5:20:58<4:01:05, 17.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.5326):  55%|█████▍    | 990/1812 [5:24:01<3:22:29, 14.78s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.5159):  55%|█████▌    | 1000/1812 [5:27:18<4:38:37, 20.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.5500):  56%|█████▌    | 1010/1812 [5:31:29<6:16:10, 28.14s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.5015):  56%|█████▋    | 1020/1812 [5:34:40<3:44:30, 17.01s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.3154):  57%|█████▋    | 1030/1812 [5:37:48<3:54:35, 18.00s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.2675):  57%|█████▋    | 1040/1812 [5:41:46<5:03:44, 23.61s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.5815):  58%|█████▊    | 1050/1812 [5:44:52<4:35:37, 21.70s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.5740):  58%|█████▊    | 1060/1812 [5:48:03<4:36:02, 22.02s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.6316):  59%|█████▉    | 1070/1812 [5:51:22<3:43:25, 18.07s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.2885):  60%|█████▉    | 1080/1812 [5:55:03<4:33:16, 22.40s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.1433):  60%|██████    | 1090/1812 [5:58:28<4:43:24, 23.55s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.3425):  61%|██████    | 1100/1812 [6:01:12<3:33:53, 18.02s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.6440):  61%|██████▏   | 1110/1812 [6:04:30<4:44:48, 24.34s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.2386):  62%|██████▏   | 1120/1812 [6:08:43<5:12:11, 27.07s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 3.3683):  62%|██████▏   | 1130/1812 [6:11:38<4:02:11, 21.31s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.2072):  63%|██████▎   | 1140/1812 [6:14:49<3:28:31, 18.62s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.5693):  63%|██████▎   | 1150/1812 [6:17:57<3:18:12, 17.96s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.4388):  64%|██████▍   | 1160/1812 [6:21:06<3:27:51, 19.13s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.3563):  65%|██████▍   | 1170/1812 [6:23:51<3:10:58, 17.85s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.6261):  65%|██████▌   | 1180/1812 [6:27:17<3:19:06, 18.90s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 1.6251):  66%|██████▌   | 1190/1812 [6:30:47<3:35:50, 20.82s/it]
+
+
+
+
+
+
+
+
+
+
+Training 2/3 epoch (loss 0.1191):  66%|██████▌   | 1200/1812 [6:34:03<3:45:32, 22.11s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.4520):  67%|██████▋   | 1210/1812 [6:37:17<3:15:05, 19.44s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 1.2468):  67%|██████▋   | 1220/1812 [6:39:52<2:30:07, 15.22s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1202):  68%|██████▊   | 1230/1812 [6:42:32<3:04:37, 19.03s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.3351):  68%|██████▊   | 1240/1812 [6:46:41<3:51:47, 24.31s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.5171):  69%|██████▉   | 1250/1812 [6:49:40<2:39:53, 17.07s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.5313):  70%|██���███▉   | 1260/1812 [6:52:50<2:52:06, 18.71s/it]
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.2009):  70%|███████   | 1269/1812 [6:56:12<3:54:14, 25.88s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.4251):  71%|███████   | 1280/1812 [7:00:15<2:48:56, 19.05s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1707):  71%|███████   | 1290/1812 [7:03:41<2:43:42, 18.82s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0537):  72%|███████▏  | 1300/1812 [7:07:34<3:23:22, 23.83s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 1.1270):  72%|███████▏  | 1310/1812 [7:11:08<2:56:55, 21.15s/it]
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1507):  73%|███████▎  | 1319/1812 [7:13:20<1:50:39, 13.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1733):  73%|███████▎  | 1330/1812 [7:17:04<2:19:01, 17.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1482):  74%|███████▍  | 1340/1812 [7:19:57<2:01:46, 15.48s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.8841):  75%|███████▍  | 1350/1812 [7:23:27<2:34:45, 20.10s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.4250):  75%|███████▌  | 1360/1812 [7:26:38<2:28:44, 19.75s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0823):  76%|███████▌  | 1370/1812 [7:30:01<2:30:00, 20.36s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.3407):  76%|███████▌  | 1380/1812 [7:33:09<2:09:52, 18.04s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.2706):  77%|███████▋  | 1390/1812 [7:36:39<2:17:35, 19.56s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1075):  77%|███████▋  | 1400/1812 [7:39:35<2:06:39, 18.44s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.9263):  78%|███████▊  | 1410/1812 [7:42:42<2:14:20, 20.05s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.2819):  78%|███████▊  | 1420/1812 [7:46:02<1:50:22, 16.89s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1509):  79%|███████▉  | 1430/1812 [7:49:15<2:04:21, 19.53s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.2960):  79%|███████▉  | 1440/1812 [7:52:10<1:34:26, 15.23s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.6236):  80%|████████  | 1450/1812 [7:55:24<2:11:47, 21.84s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1412):  81%|████████  | 1460/1812 [7:59:24<2:35:43, 26.54s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.4150):  81%|████████  | 1470/1812 [8:02:50<2:19:29, 24.47s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1795):  82%|████████▏ | 1480/1812 [8:07:09<2:29:39, 27.05s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1406):  82%|████████▏ | 1490/1812 [8:09:53<1:35:14, 17.75s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0995):  83%|████████▎ | 1500/1812 [8:13:09<1:37:02, 18.66s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0809):  83%|████████▎ | 1510/1812 [8:16:18<1:34:23, 18.75s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.7454):  84%|████████▍ | 1520/1812 [8:19:01<1:22:08, 16.88s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1526):  84%|████████▍ | 1530/1812 [8:21:49<1:33:03, 19.80s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.4201):  85%|████████▍ | 1540/1812 [8:24:36<1:11:43, 15.82s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1581):  86%|████████▌ | 1550/1812 [8:28:03<1:49:36, 25.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.2343):  86%|████████▌ | 1560/1812 [8:31:10<1:23:34, 19.90s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.3496):  87%|████████▋ | 1570/1812 [8:34:36<1:13:02, 18.11s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.2333):  87%|████████▋ | 1580/1812 [8:37:54<1:19:15, 20.50s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0774):  88%|████████▊ | 1590/1812 [8:41:14<1:10:06, 18.95s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.7136):  88%|████████▊ | 1600/1812 [8:44:06<1:08:22, 19.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1794):  89%|████████▉ | 1610/1812 [8:47:52<1:24:00, 24.95s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0726):  89%|████████▉ | 1620/1812 [8:51:42<1:08:22, 21.36s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1578):  90%|████████▉ | 1630/1812 [8:54:42<1:08:44, 22.66s/it]
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0878):  90%|█████████ | 1639/1812 [8:57:47<56:50, 19.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.3154):  91%|█████████ | 1650/1812 [9:01:39<53:37, 19.86s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0431):  92%|███��█████▏| 1660/1812 [9:04:45<47:04, 18.58s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.2098):  92%|█████████▏| 1670/1812 [9:08:21<50:39, 21.40s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1670):  93%|█████████▎| 1680/1812 [9:11:52<46:14, 21.02s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.2508):  93%|█████████▎| 1690/1812 [9:14:59<36:51, 18.13s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.2887):  94%|█████████▍| 1700/1812 [9:18:09<30:03, 16.10s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0991):  94%|█████████▍| 1710/1812 [9:21:02<30:54, 18.18s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0438):  95%|█████████▍| 1720/1812 [9:25:09<35:44, 23.31s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0824):  95%|█████████▌| 1730/1812 [9:28:26<24:32, 17.96s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0850):  96%|█████████▌| 1740/1812 [9:31:48<24:37, 20.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.0191):  97%|█████████▋| 1750/1812 [9:34:53<19:15, 18.63s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.4226):  97%|█████████▋| 1760/1812 [9:38:05<17:47, 20.52s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1399):  98%|█████████▊| 1770/1812 [9:40:31<08:16, 11.83s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1972):  98%|█████████▊| 1780/1812 [9:44:17<10:57, 20.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.2154):  99%|█████████▉| 1790/1812 [9:47:45<07:39, 20.88s/it]
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1399):  99%|█████████▉| 1800/1812 [9:50:39<03:33, 17.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+Training 3/3 epoch (loss 0.1139): 100%|█████████▉| 1810/1812 [9:54:08<00:35, 17.66s/it]
+
+
+Training 3/3 epoch (loss 0.0352): 100%|██████████| 1812/1812 [9:54:45<00:00, 19.69s/it]
+Saving model to "../outputs/dpo_tiv2t_10k_baseline" ...
+Saving 16-bit model...
+[2024-09-22 16:28:28,884] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1812 is about to be saved!
+[2024-09-22 16:28:28,885] [INFO] [engine.py:3591:save_16bit_model] Saving model weights to ../outputs/dpo_tiv2t_10k_baseline/pytorch_model.bin, tag: global_step1812
+[2024-09-22 16:28:28,885] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/dpo_tiv2t_10k_baseline/pytorch_model.bin...
+[2024-09-22 16:28:43,063] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/dpo_tiv2t_10k_baseline/pytorch_model.bin.
+[2024-09-22 16:28:43,063] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1812 is ready now!
+Model saved!
+Saving 16-bit model...
+[2024-09-22 16:28:51,576] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1812 is about to be saved!
+[2024-09-22 16:28:51,577] [INFO] [engine.py:3591:save_16bit_model] Saving model weights to ../outputs/dpo_tiv2t_10k_baseline/pytorch_model.bin, tag: global_step1812
+[2024-09-22 16:28:51,577] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/dpo_tiv2t_10k_baseline/pytorch_model.bin...
+[2024-09-22 16:29:07,128] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/dpo_tiv2t_10k_baseline/pytorch_model.bin.
+[2024-09-22 16:29:07,128] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1812 is ready now!
+Model saved!

wandb/run-20240922_063327-rptdqsqq/files/requirements.txt

@@ -0,0 +1,325 @@
+Brotli==1.1.0
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Werkzeug==3.0.3
+about-time==4.2.1
+absl-py==2.1.0
+accelerate==0.34.0
+aiofiles==23.2.1
+aiohttp-retry==2.8.3
+aiohttp==3.9.5
+aiosignal==1.3.1
+align-anything==0.0.1.dev0
+align-anything==0.0.1.dev0
+alive-progress==3.1.5
+amqp==5.2.0
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+appdirs==1.4.4
+asttokens==2.4.1
+asyncssh==2.16.0
+atpublic==5.0
+attrs==23.2.0
+audioread==3.0.1
+autocommand==2.2.2
+backoff==2.2.1
+backports.tarfile==1.2.0
+billiard==4.2.0
+boto3==1.35.9
+botocore==1.35.9
+cachetools==5.4.0
+celery==5.4.0
+certifi==2024.7.4
+cffi==1.16.0
+cfgv==3.4.0
+charset-normalizer==3.3.2
+click-didyoumean==0.3.1
+click-plugins==1.1.1
+click-repl==0.3.0
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorama==0.4.6
+comm==0.2.2
+commonmark==0.9.1
+configobj==5.0.8
+contourpy==1.2.1
+cryptography==43.0.0
+cycler==0.12.1
+dacite==1.6.0
+dagshub-annotation-converter==0.1.0
+dagshub==0.3.35
+dataclasses-json==0.6.7
+datasets==2.20.0
+debugpy==1.8.2
+decorator==4.4.2
+deepspeed==0.14.4
+dictdiffer==0.9.0
+diffusers==0.29.2
+dill==0.3.8
+diskcache==5.6.3
+distlib==0.3.8
+distro==1.9.0
+dnspython==2.6.1
+docker-pycreds==0.4.0
+docstring_parser==0.16
+dpath==2.2.0
+dulwich==0.22.1
+dvc-data==3.16.5
+dvc-http==2.32.0
+dvc-objects==5.1.0
+dvc-render==1.0.2
+dvc-studio-client==0.21.0
+dvc-task==0.4.0
+dvc==3.55.1
+einops==0.8.0
+email_validator==2.2.0
+entrypoints==0.4
+exceptiongroup==1.2.2
+executing==2.0.1
+fastapi-cli==0.0.4
+fastapi==0.111.1
+ffmpy==0.3.3
+filelock==3.15.4
+fire==0.6.0
+flatten-dict==0.4.2
+flufl.lock==8.1.0
+fonttools==4.53.1
+frozenlist==1.4.1
+fsspec==2024.5.0
+funcy==2.0
+fusepy==3.0.1
+gitdb==4.0.11
+gmpy2==2.1.5
+gql==3.5.0
+gradio==4.39.0
+gradio_client==1.1.1
+gradio_multimodalchatbot==0.0.1
+grandalf==0.8
+grapheme==0.6.0
+graphql-core==3.2.3
+grpcio==1.65.1
+gto==1.7.1
+h11==0.14.0
+h2==4.1.0
+hjson==3.1.0
+hpack==4.0.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.2
+huggingface-hub==0.24.3
+hydra-core==1.3.2
+hyperframe==6.0.1
+identify==2.6.0
+idna==3.7
+imageio-ffmpeg==0.5.1
+imageio==2.35.1
+importlib_metadata==8.0.0
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+importlib_resources==6.4.0
+inflect==7.3.1
+iniconfig==2.0.0
+interegular==0.3.3
+ipykernel==6.29.5
+ipython==8.26.0
+iterative-telemetry==0.0.8
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+jedi==0.19.1
+jmespath==1.0.1
+joblib==1.4.2
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+jupyter_client==8.6.2
+jupyter_core==5.7.2
+kiwisolver==1.4.5
+kombu==5.4.0
+lark==1.1.9
+lazy_loader==0.4
+librosa==0.10.2.post1
+llamafactory==0.8.4.dev0
+llamafactory==0.8.4.dev0
+llvmlite==0.43.0
+lm-format-enforcer==0.10.3
+lxml==5.3.0
+markdown-it-py==3.0.0
+marshmallow==3.22.0
+matplotlib-inline==0.1.7
+matplotlib==3.9.1
+mdurl==0.1.2
+mmsg==0.1.dev14+gbd2b056.d20240803
+more-itertools==10.3.0
+moviepy==1.0.3
+mpi4py==4.0.0
+mpmath==1.3.0
+msgpack==1.0.8
+multidict==6.0.5
+multiprocess==0.70.16
+mypy-extensions==1.0.0
+nest_asyncio==1.6.0
+networkx==3.3
+ninja==1.11.1.1
+nodeenv==1.9.1
+numba==0.60.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-ml-py==12.535.161
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+nvitop==1.3.2
+omegaconf==2.3.0
+openai==1.37.1
+optree==0.12.1
+ordered-set==4.1.0
+orjson==3.10.6
+outlines==0.0.47.dev37+g26e2934
+packaging==24.1
+packaging==24.1
+pandas==2.2.2
+parso==0.8.4
+pathspec==0.12.1
+pathvalidate==3.0.0
+peft==0.12.0
+pexpect==4.9.0
+pickleshare==0.7.5
+pillow==10.4.0
+pip==24.0
+platformdirs==3.11.0
+platformdirs==4.2.2
+pluggy==1.5.0
+pooch==1.8.2
+pre-commit==3.8.0
+proglog==0.1.10
+prometheus-fastapi-instrumentator==7.0.0
+prometheus_client==0.20.0
+prompt_toolkit==3.0.47
+protobuf==4.25.4
+psutil==6.0.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+py-cpuinfo==9.0.0
+pyairports==2.1.1
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyav==13.0.0
+pycountry==24.6.1
+pycparser==2.22
+pydantic==2.8.2
+pydantic_core==2.20.1
+pydot==3.0.1
+pydub==0.25.1
+pygit2==1.15.1
+pygtrie==2.5.0
+pyparsing==3.1.2
+pytest==8.3.2
+python-dateutil==2.9.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2024.1
+pyzmq==26.0.3
+ranger-fm==1.9.3
+ray==2.34.0
+referencing==0.35.1
+regex==2024.7.24
+requests-toolbelt==1.0.0
+requests==2.32.3
+rfc3986==1.5.0
+rich==13.1.0
+rpds-py==0.19.1
+ruamel.yaml.clib==0.2.8
+ruamel.yaml==0.18.6
+ruff==0.5.5
+s3transfer==0.10.2
+safetensors==0.4.3
+scikit-learn==1.5.1
+scipy==1.14.0
+scmrepo==3.3.7
+semantic-version==2.10.0
+semver==3.0.2
+sentencepiece==0.2.0
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==71.0.4
+shellingham==1.5.4
+shortuuid==1.0.13
+shtab==1.7.1
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+soundfile==0.12.1
+soxr==0.4.0
+sqltrie==0.11.1
+sse-starlette==2.1.3
+stack-data==0.6.2
+starlette==0.37.2
+sympy==1.13.0
+tabulate==0.9.0
+tenacity==8.2.3
+tensorboard-data-server==0.7.2
+tensorboard==2.17.0
+term-image==0.7.2
+termcolor==2.4.0
+threadpoolctl==3.5.0
+tiktoken==0.7.0
+tokenizers==0.19.1
+tomli==2.0.1
+tomlkit==0.12.0
+torch==2.3.1
+torch==2.4.0
+torchaudio==2.4.0
+torchvision==0.18.1
+torchvision==0.19.0
+tornado==6.4.1
+tqdm==4.66.4
+traitlets==5.14.3
+transformers==4.45.0.dev0
+treelib==1.6.4
+triton==2.3.1
+triton==3.0.0
+trl==0.9.6
+typeguard==4.3.0
+typer==0.12.3
+typing-inspect==0.9.0
+typing_extensions==4.12.2
+typing_extensions==4.12.2
+tyro==0.8.10
+tzdata==2024.1
+urllib3==2.2.2
+uvicorn==0.30.3
+uvloop==0.19.0
+vine==5.1.0
+virtualenv==20.26.3
+vllm-flash-attn==2.5.9.post1
+vllm==0.5.3.post1
+voluptuous==0.15.2
+wandb==0.17.5
+watchfiles==0.22.0
+wcwidth==0.2.13
+websockets==11.0.3
+wheel==0.43.0
+wheel==0.43.0
+xformers==0.0.27
+xxhash==3.4.1
+yarl==1.9.4
+zc.lockfile==3.0.post1
+zipp==3.19.2
+zipp==3.19.2
+zstandard==0.23.0

wandb/run-20240922_063327-rptdqsqq/files/wandb-metadata.json

@@ -0,0 +1,1215 @@
+{
+    "os": "Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35",
+    "python": "3.11.9",
+    "heartbeatAt": "2024-09-21T22:33:28.392138",
+    "startedAt": "2024-09-21T22:33:27.553001",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--local_rank=0",
+        "--model_name_or_path",
+        "/aifs4su/yaodong/models/Qwen2-VL-7B-Instruct",
+        "--train_datasets",
+        "/aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted",
+        "--output_dir",
+        "../outputs/dpo_tiv2t_10k_baseline",
+        "--train_data_files",
+        "extracted_preference_10k_washed.json",
+        "--train_template",
+        "NExTQA_preference",
+        "--train_split",
+        "train",
+        "--per_device_train_batch_size",
+        "1",
+        "--per_device_eval_batch_size",
+        "1",
+        "--gradient_accumulation_steps",
+        "1"
+    ],
+    "state": "running",
+    "program": "-m align_anything.trainers.tiv_to_t.dpo",
+    "codePathLocal": null,
+    "git": {
+        "remote": "[email protected]:htlou/align-anything.git",
+        "commit": "9eff99b0ee34f7265b8a2e1824650634c898f654"
+    },
+    "email": "[email protected]",
+    "root": "/aifs4su/yaodong/projects/hantao/dev_cham/align-anything",
+    "host": "dgx-081",
+    "username": "yangyaodong",
+    "executable": "/aifs4su/yaodong/miniconda3/envs/hantao_tiv/bin/python3.11",
+    "cpu_count": 112,
+    "cpu_count_logical": 224,
+    "cpu_freq": {
+        "current": 2064.275857142857,
+        "min": 800.0,
+        "max": 3800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.998,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.998,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.001,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.998,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.993,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.907,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 1758.8549690246582,
+            "used": 58.09877014160156
+        }
+    },
+    "gpu": "NVIDIA H800",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        }
+    ],
+    "memory": {
+        "total": 2015.5615730285645
+    }
+}

wandb/run-20240922_063327-rptdqsqq/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/step": 1812, "train/loss": 0.035232577472925186, "train/reward": -9.132875442504883, "train/better_sample_reward": -1.052369475364685, "train/worse_sample_reward": -8.08050537109375, "train/reward_accuracy": 1.0, "train/reward_margin": 7.028136253356934, "train/lr": 0.0, "train/epoch": 3.0, "_timestamp": 1726993700.3175497, "_runtime": 35692.75268268585, "_step": 1812, "_wandb": {"runtime": 35739}}

wandb/run-20240922_063327-rptdqsqq/logs/debug.log

@@ -0,0 +1,36 @@
+2024-09-22 06:33:27,559 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Configure stats pid to 1338204
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Loading settings from /home/yangyaodong/.config/wandb/settings
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Loading settings from /aifs4su/yaodong/projects/hantao/dev_cham/align-anything/scripts/wandb/settings
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***'}
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-09-22 06:33:27,560 WARNING MainThread:1338204 [wandb_setup.py:_flush():76] Could not find program at -m align_anything.trainers.tiv_to_t.dpo
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.tiv_to_t.dpo'}
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:_log_setup():529] Logging user logs to ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/logs/debug.log
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:_log_setup():530] Logging internal logs to ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_063327-rptdqsqq/logs/debug-internal.log
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:init():569] calling init triggers
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1.0, 'per_device_eval_batch_size': 1.0, 'gradient_accumulation_steps': 1.0, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.01, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'regularization': 0.001, 'scale_coeff': 0.1, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False}, 'data_cfgs': {'train_datasets': '/aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted', 'train_template': 'NExTQA_preference', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'extracted_preference_10k_washed.json', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'dpo', 'output_dir': '../outputs/dpo_tiv2t_10k_baseline', 'cache_dir': None, 'save_interval': 100000}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/models/Qwen2-VL-7B-Instruct', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:init():619] starting backend
+2024-09-22 06:33:27,560 INFO    MainThread:1338204 [wandb_init.py:init():623] setting up manager
+2024-09-22 06:33:27,561 INFO    MainThread:1338204 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-09-22 06:33:27,564 INFO    MainThread:1338204 [wandb_init.py:init():631] backend started and connected
+2024-09-22 06:33:27,566 INFO    MainThread:1338204 [wandb_init.py:init():720] updated telemetry
+2024-09-22 06:33:27,587 INFO    MainThread:1338204 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-09-22 06:33:28,008 INFO    MainThread:1338204 [wandb_run.py:_on_init():2435] communicating current version
+2024-09-22 06:33:28,205 INFO    MainThread:1338204 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.18.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-09-22 06:33:28,205 INFO    MainThread:1338204 [wandb_init.py:init():804] starting run threads in backend
+2024-09-22 06:33:34,310 INFO    MainThread:1338204 [wandb_run.py:_console_start():2413] atexit reg
+2024-09-22 06:33:34,310 INFO    MainThread:1338204 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-09-22 06:33:34,310 INFO    MainThread:1338204 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-09-22 06:33:34,310 INFO    MainThread:1338204 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-09-22 06:33:34,313 INFO    MainThread:1338204 [wandb_init.py:init():847] run started, returning control to user process
+2024-09-22 16:29:07,169 INFO    MainThread:1338204 [wandb_run.py:_finish():2107] finishing run htlou/align-anything/rptdqsqq
+2024-09-22 16:29:07,171 INFO    MainThread:1338204 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0
+2024-09-22 16:29:07,172 INFO    MainThread:1338204 [wandb_run.py:_restore():2352] restore
+2024-09-22 16:29:07,172 INFO    MainThread:1338204 [wandb_run.py:_restore():2358] restore done
+2024-09-22 16:29:15,801 INFO    MainThread:1338204 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-09-22 16:29:15,802 INFO    MainThread:1338204 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-09-22 16:29:15,809 INFO    MainThread:1338204 [wandb_run.py:_footer_sync_info():3975] logging synced files

wandb/run-20240922_063327-rptdqsqq/run-rptdqsqq.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ef92868525650ac6091a54236f14dbabca26113a005c7965c09bf17e5773290
+size 11735552