diff --git a/.gitattributes b/.gitattributes index 669330baa49b824d5240b194a242302222e9208b..5df39ea16d3119275c097c86e393e7dcc7bd9713 100644 --- a/.gitattributes +++ b/.gitattributes @@ -996,3 +996,56 @@ checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_se checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_92000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_92000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/run-mlbv5b7s.wandb filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/run-0k7f5yzn.wandb filter=lfs diff=lfs merge=lfs -text +EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/run-yx27lyn0.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0090000.pt b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0090000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7908f358f879076fa71b5f029ed56d9db23579e2 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0090000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:855a3f8afd6c57f840c77e1dcee02b8fc077dea9b00cbf70d1932d53bc499933 +size 19774276767 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e1d3a494c42226b8258b7df376127f3796fd5d46 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65ad3640b3b55e2ca07e12fdd06961308b3109c89de6b1dd49385789c974d38e +size 1069767 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7bc04c73b591a8509d97f292fa9a08692b685c1a --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93c3fe9b1c63a97d76181f9fff11cbb42663c426fd24d8ba00a50ea758dbd92d +size 908556 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0a5a21ba54880b19081d3b0487df8a5e84614e27 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d468e9ef1c49875f1aecf2713488696b7133fe34d0cdea1bf3745d6832b9ae0 +size 1031007 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..50f4376ad1d382b60fb606efc9faba2ee328f146 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51ed27cda10989c91b4dd20dd44c0d02ae536d19d9cc65041fb62d388ecb5743 +size 1063700 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bed0d88fa06bb44190c7fed5e940cfbb9ddd5d06 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8124bf40a6dd878dd995f86a1ecaf1c98b9f8eeca6cb0ce8223fcb259d789553 +size 905524 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..53b1e8dfa571664d2cd256b0d3887503bd53c0c2 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db36dc28279f88311d3088046c33d55a9ca11ca30a32dfb5cea6f7be6b419bd5 +size 992311 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4f723df01cea11d8646a01887e494665b51ac93d --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884621c117651f0ca4aea21f3a7292c50a4f810d7580f9230cffa153a3f0dda4 +size 1096682 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..01e348722dc759cf79260cb601f3644e3b3f5615 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc844515b8c03c2b75aa9669c802f2c1e09d7e018022b8f0b4ec39c478e5b3f4 +size 899056 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ce1566f3dcb0c42be8ee9af5965a60b6d6170a81 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1aa510cc481706dc1f3ba31acaf212d0e0d6edcdc2528e66cf22cb5642762e4e +size 1031221 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f8ae839ffe18db4b7c4bec6b524369ad41835814 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:329edfac2738e94f4043029d3f43437d821b5b4297927acf0e15c5fc524c24f0 +size 1070907 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..30a7015456f66d092661bc643c6a7fd22e2984c6 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eee9b7b3294a493a24f7fc313455906ea38a403b125bb7326bff12ed7e8c5731 +size 908485 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4462fd80d7de3b14bfc79d3ed07b85599fd1a9a7 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d17cbad262d65c5c22d516535c57df49571c7603c7afbc06ca24c3393acd764e +size 1035683 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..739880c4d3d33672777c93e3e2173c17e9e5f5a5 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8be4f325f89687af98d4365b01e414cf261ff1f5c10b851a24e708c8b82b7f00 +size 1061517 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4122e72d8a50895fe46a0a01c06a6542f53d25db --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bb932d1bfdabb040f97d81b9b3be3bf67a7709991310642565c205231d280bd +size 906377 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1966221745673f6e5d69aff810f10529df26bc54 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc3705546aa053920ef30d905d70d362c897b974d7beee2d83abe2ea735ae60f +size 1043039 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..81667657f830dc8dea66c0747c99c9f585650144 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55c7d49ee175e1ce570d9155788d628509bfd28866f3d71a0369ce28bf685cb7 +size 1064717 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c8d5e1c46e29809a702d57776ffeaba5dc82eac4 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3026d483e545784125b9c3b9e8cc739a720df87c6c22d512d302a52c89398e5 +size 913927 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..89a1bb90d9b44f83e927fdfc87633d69c454b874 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fa1bc0dedb11e89dbeed90c29a1b47f306d00967455a3b319d314dacaf191fd +size 1017171 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f270eb6d8ae6131a71be1fe3c4ce7549e6b8f122 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef04dd36838dad2caa4ad26171c55b5ea0258fee4c5c7df0ac23461be268bf33 +size 1095383 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5d498d6fc91215b6c02aa4daa959483ac949d768 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74e73f4920a033ad06e66f96c5d9f37740b9aa2c021351880befbe2ac9c0b7e0 +size 905785 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1657635c4a175fe0af50e893c72802af4cbb55cc --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c524c79674aa545b2b8c2fa217011b77df012b4e6623e2dbed8c6a49eec8038b +size 1006189 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..25b6b8b66078fc71c55d0fe425ca9094e12c7915 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b9c62104662d448f68a9616ac55810ec8cb13ca47cbec1dad591546ace20bdf +size 1069445 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..016616bb1f9b57b6b9feaa87995a80da492042aa --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c2576b5d897fb73dba4d439122b468c8aaea240c03f41eff4abdf8bfae50a9e +size 905705 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4b32e597e64e66dfa5140a48d57816d143fe05c7 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ebf5f1ab408c5682bf3b85f6ff99946b68e46c629b27043282f8edda5347342 +size 1041513 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..257099186181dc36e5001bebf35967dcdcba78ed --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3834bd6d8bfc2ab68a7d74d2c4f4442b88cfb308302edcfff5ecaf2ad6650b23 +size 1077175 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f82b2a3991abbff21ac9de61bebe9ec04c4b2bb4 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa9b41027298f8086dd9287a4b1f802b539766704641e527788f87c299d7ba04 +size 890792 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b34a0a590f898ddf33065966542382f007faab3e --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:727ab8f15b0e5fc94682ed8d56310782dbe9613d53b4bda9d861928faa725fc5 +size 1012065 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1ab62cdbea0ac549ab91084b253db6d653714410 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c6bdc3dc9cb35c49daae702125f639f7385c661e474a6aa2f544295717b3efe +size 1134964 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..759eec13167a37da50e5814148324922f8882e01 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94380fd9d8b4ded43ef1cf8c895b7457bdb813876502a0393d728e609689a374 +size 904412 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d5789ec2314250d12c3efcb67384c0287aaeee7d --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9abcc7319b66cd9d96b1c1a8e80c745ddb3b9b5a15984493fc3b7846deabc0b0 +size 1001615 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d3ab82767b3be83905ee0297f15e77447be3dbff --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb1302a6386c1e134f75b29f6d267bb4dda7df0ddd910c365c91c80d3b1f311 +size 1055659 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..81ccf2460c8e69d16d139addb9a69a9c2ab10c7a --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e84ed079ab956d36eec6a4e639970d50974de617d99af31c1993ba0a7596051 +size 924637 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a9f98f1f85c9c25bdf6f78e10467a308995bd85a --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c12de4693dc8540bbf890bd5e6b2664017a84b369b15d2ef79e5d0c76e88aef5 +size 1011783 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1f7fa9daffd6aa8db98b6c5f48e26c3fde5bff63 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efafb9afd01f94650101a133b59330cd12d2d880076cdc2ccaf176f89e94322b +size 1073220 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..eea8d318e392239110c00bd78cf4c804a1a4b238 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45669781455fa200c578d1897140ccad2962c5f0c213e85ed96f0b645092f803 +size 882184 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4456ab7e3ae69765d9abba9047313c5dd2eb2c03 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3444aceabb2de57c617e82320b463209ab57ec506662038776fe6ba7a1cfe868 +size 1026337 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..539df19af6a4420398b48ddb0f503f1d7380022b --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ed196c7c1ba93f2f92f37de9a2a11a7bf0f1f808a0dbb06ed423109fff1ebb0 +size 1048317 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..53b203bce7c8456f9a2979eff5d6590dd4624286 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37b5ad01f85ef5a79aeb540957489de024673baa433f6850c7ea214fef7c7340 +size 897121 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d276daa839bd167e760cc8be34a4af8ec6553975 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06dc944857a1896627dbfffcce69f3470f4dcd9e129b82e436a4d02aced1ce5a +size 1016656 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..76b306c4f6aa02412573be4f4031e9d898fe52ba --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84be39dca5ee36e4247759c8f10a8bf80b03b9887558df9911f1625b3fd3f238 +size 1095988 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..25bddb0686f0af82bc616939a59abd5d663b8135 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20e30593863c3d97da3b73cb2717afd329d55f2f40b19b96e62b0b27ada356e9 +size 900663 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..255648a7c763f03d02b03e40fa2c150acff60f88 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aafc24a81a29e1a849e3e0111fd016f007178759262583e1b6365ff5faf32e78 +size 1006321 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..42d79483d025e7d4e319964cd02a9ff869a83b03 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e652a63614ff8c58865baad7323085ef62e97203a1ee7fa9c5fc150a84e4cecb +size 1103260 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b872932c20ece2d2ec4f9cff1eedfc4dce6cf665 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b07621545b189295bce8de3096cce3ca6916514ed314d6aa7ae5b8d3bdc388d5 +size 895525 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ca0f1fee22de82ccf309e79ce73e1078b5687b5a --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97a6880c9c4a1a31c2836ac8f577588b4cc0714781e4d2123106016770d685b6 +size 1001603 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..925bbe57511626cf580b2c80dd6c76d40829e84d --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3078e5dfca3c19a33bfddf713e97e6048969cdca493b83910db34a08aa3f354 +size 1071692 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2afc7b6e3eccc5208170545b94115750b55c71d5 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf46fe8934d2f040901d9b2e25ea0ed80a6f46e0ee9484cfc4bf0f812d3830b0 +size 908150 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8143931e90ca1d8400b11dc50c41aba47083761f --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1101cb729a693e260b733fa5ce510e5167099dd142d42bd76dc3c34a3711749d +size 1005495 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f4688a01aa708041cf6dab7a9f23b0f83c9414c8 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbc01c99a5453c018b157821f1dbb14a6bd130cebc9c3db4f62153077b3f93ce +size 1109512 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ab85a5ffc559dc8331e54b0fde148842ca3270d6 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbffca54bff2b0a7b65cab36b07b1d32bc3307a2bacbabc22d04c6210e716862 +size 895919 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3041516c838e5c1c42cb518470cc86ec5c46acb1 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5b2c8d8ce06bb8c8eea6c2476f67fb679da641a13bec992bd1e276c4b19f30d +size 1072514 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/log.txt b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..4411312ad89c05afa9cad15925fbcedd6b874abd --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/log.txt @@ -0,0 +1,1555 @@ +[2025-04-24 14:28:41] Experiment directory created at checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL +[2025-04-24 14:28:41] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_trained_raw_cleaned.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0056000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=5e-05, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_val_raw_cleaned.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/instructblip-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=200, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='instructblip', do_recovery=True, no_replace=False, resume=True, dreambench_eval=False, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection=None, mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-24 14:28:41] Starting rank=0, seed=0, world_size=8. +[2025-04-24 14:28:41] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 14:28:41] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 14:28:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 14:30:22] GPT Parameters: 3,174,417,408 +[2025-04-24 14:30:22] num decayed parameter tensors: 637, with 3,173,270,016 parameters +[2025-04-24 14:30:22] num non-decayed parameter tensors: 644, with 1,147,392 parameters +[2025-04-24 14:30:22] using fused AdamW: True +[2025-04-24 14:30:54] Dataset contains 2,571,360 images +[2025-04-24 14:30:54] Train iters 91834 , warmup 4591.7, len of loader 45917 +[2025-04-24 14:31:18] Resume training from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0056000.pt +[2025-04-24 14:31:18] Initial state: steps=56000, epochs=1 +[2025-04-24 14:31:18] compiling the model... (may take several minutes) +[2025-04-24 14:31:19] freeze the vit +[2025-04-24 14:31:19] ***** total param is 3174417408 ***** +[2025-04-24 14:31:19] ***** total trained param is 2188465152 ***** +[2025-04-24 14:31:24] Training for 2 epochs... +[2025-04-24 14:31:24] Beginning epoch 1... +[2025-04-24 15:15:15] (step=0056025) Train Loss: 6.2975, Train Steps/Sec: 0.01 +[2025-04-24 15:19:42] (step=0056050) Train Loss: 6.3270, Train Steps/Sec: 0.09 +[2025-04-24 15:20:17] (step=0056075) Train Loss: 6.3124, Train Steps/Sec: 0.71 +[2025-04-24 15:20:45] (step=0056100) Train Loss: 6.2960, Train Steps/Sec: 0.90 +[2025-04-24 15:21:13] (step=0056125) Train Loss: 6.3331, Train Steps/Sec: 0.91 +[2025-04-24 15:21:40] (step=0056150) Train Loss: 6.3175, Train Steps/Sec: 0.91 +[2025-04-24 15:22:08] (step=0056175) Train Loss: 6.3137, Train Steps/Sec: 0.90 +[2025-04-24 15:22:36] (step=0056200) Train Loss: 6.3111, Train Steps/Sec: 0.90 +[2025-04-24 15:23:11] (step=0056225) Train Loss: 6.3284, Train Steps/Sec: 0.71 +[2025-04-24 15:23:39] (step=0056250) Train Loss: 6.3305, Train Steps/Sec: 0.90 +[2025-04-24 15:24:06] (step=0056275) Train Loss: 6.3283, Train Steps/Sec: 0.91 +[2025-04-24 15:24:34] (step=0056300) Train Loss: 6.3770, Train Steps/Sec: 0.90 +[2025-04-24 15:25:02] (step=0056325) Train Loss: 6.3261, Train Steps/Sec: 0.90 +[2025-04-24 15:25:29] (step=0056350) Train Loss: 6.3704, Train Steps/Sec: 0.91 +[2025-04-24 15:25:57] (step=0056375) Train Loss: 6.3315, Train Steps/Sec: 0.90 +[2025-04-24 15:26:25] (step=0056400) Train Loss: 6.3438, Train Steps/Sec: 0.90 +[2025-04-24 15:26:53] (step=0056425) Train Loss: 6.3129, Train Steps/Sec: 0.90 +[2025-04-24 15:27:20] (step=0056450) Train Loss: 6.3091, Train Steps/Sec: 0.90 +[2025-04-24 15:27:48] (step=0056475) Train Loss: 6.3124, Train Steps/Sec: 0.91 +[2025-04-24 15:28:16] (step=0056500) Train Loss: 6.3059, Train Steps/Sec: 0.90 +[2025-04-24 15:28:43] (step=0056525) Train Loss: 6.3558, Train Steps/Sec: 0.91 +[2025-04-24 15:29:11] (step=0056550) Train Loss: 6.3492, Train Steps/Sec: 0.90 +[2025-04-24 15:29:39] (step=0056575) Train Loss: 6.3300, Train Steps/Sec: 0.91 +[2025-04-24 15:30:07] (step=0056600) Train Loss: 6.2586, Train Steps/Sec: 0.90 +[2025-04-24 15:30:34] (step=0056625) Train Loss: 6.3573, Train Steps/Sec: 0.90 +[2025-04-24 15:31:02] (step=0056650) Train Loss: 6.2588, Train Steps/Sec: 0.90 +[2025-04-24 15:31:30] (step=0056675) Train Loss: 6.2753, Train Steps/Sec: 0.90 +[2025-04-24 15:31:57] (step=0056700) Train Loss: 6.2903, Train Steps/Sec: 0.90 +[2025-04-24 15:32:25] (step=0056725) Train Loss: 6.3366, Train Steps/Sec: 0.91 +[2025-04-24 15:32:53] (step=0056750) Train Loss: 6.2722, Train Steps/Sec: 0.90 +[2025-04-24 15:33:21] (step=0056775) Train Loss: 6.3674, Train Steps/Sec: 0.91 +[2025-04-24 15:33:48] (step=0056800) Train Loss: 6.3090, Train Steps/Sec: 0.90 +[2025-04-24 15:34:16] (step=0056825) Train Loss: 6.2719, Train Steps/Sec: 0.90 +[2025-04-24 15:34:44] (step=0056850) Train Loss: 6.3305, Train Steps/Sec: 0.90 +[2025-04-24 15:35:12] (step=0056875) Train Loss: 6.3494, Train Steps/Sec: 0.90 +[2025-04-24 15:35:39] (step=0056900) Train Loss: 6.3308, Train Steps/Sec: 0.90 +[2025-04-24 15:36:07] (step=0056925) Train Loss: 6.3525, Train Steps/Sec: 0.90 +[2025-04-24 15:36:35] (step=0056950) Train Loss: 6.3302, Train Steps/Sec: 0.90 +[2025-04-24 15:37:03] (step=0056975) Train Loss: 6.3063, Train Steps/Sec: 0.90 +[2025-04-24 15:37:30] (step=0057000) Train Loss: 6.2988, Train Steps/Sec: 0.90 +[2025-04-24 15:37:58] (step=0057025) Train Loss: 6.3264, Train Steps/Sec: 0.90 +[2025-04-24 15:38:26] (step=0057050) Train Loss: 6.3505, Train Steps/Sec: 0.91 +[2025-04-24 15:38:53] (step=0057075) Train Loss: 6.3634, Train Steps/Sec: 0.90 +[2025-04-24 15:39:21] (step=0057100) Train Loss: 6.3218, Train Steps/Sec: 0.90 +[2025-04-24 15:39:49] (step=0057125) Train Loss: 6.3056, Train Steps/Sec: 0.91 +[2025-04-24 15:40:16] (step=0057150) Train Loss: 6.2911, Train Steps/Sec: 0.90 +[2025-04-24 15:40:44] (step=0057175) Train Loss: 6.2883, Train Steps/Sec: 0.90 +[2025-04-24 15:41:12] (step=0057200) Train Loss: 6.3485, Train Steps/Sec: 0.90 +[2025-04-24 15:41:40] (step=0057225) Train Loss: 6.3157, Train Steps/Sec: 0.90 +[2025-04-24 15:42:08] (step=0057250) Train Loss: 6.3210, Train Steps/Sec: 0.90 +[2025-04-24 15:42:35] (step=0057275) Train Loss: 6.2788, Train Steps/Sec: 0.90 +[2025-04-24 15:43:03] (step=0057300) Train Loss: 6.2808, Train Steps/Sec: 0.89 +[2025-04-24 15:43:31] (step=0057325) Train Loss: 6.2922, Train Steps/Sec: 0.90 +[2025-04-24 15:43:59] (step=0057350) Train Loss: 6.3324, Train Steps/Sec: 0.90 +[2025-04-24 15:44:27] (step=0057375) Train Loss: 6.2973, Train Steps/Sec: 0.90 +[2025-04-24 15:44:54] (step=0057400) Train Loss: 6.3573, Train Steps/Sec: 0.90 +[2025-04-24 15:45:22] (step=0057425) Train Loss: 6.2836, Train Steps/Sec: 0.91 +[2025-04-24 15:45:49] (step=0057450) Train Loss: 6.3219, Train Steps/Sec: 0.91 +[2025-04-24 15:46:17] (step=0057475) Train Loss: 6.3027, Train Steps/Sec: 0.90 +[2025-04-24 15:46:45] (step=0057500) Train Loss: 6.3505, Train Steps/Sec: 0.90 +[2025-04-24 15:47:13] (step=0057525) Train Loss: 6.2579, Train Steps/Sec: 0.91 +[2025-04-24 15:47:40] (step=0057550) Train Loss: 6.3601, Train Steps/Sec: 0.90 +[2025-04-24 15:48:08] (step=0057575) Train Loss: 6.3298, Train Steps/Sec: 0.91 +[2025-04-24 15:48:36] (step=0057600) Train Loss: 6.3833, Train Steps/Sec: 0.90 +[2025-04-24 15:49:03] (step=0057625) Train Loss: 6.3479, Train Steps/Sec: 0.90 +[2025-04-24 15:49:31] (step=0057650) Train Loss: 6.3426, Train Steps/Sec: 0.90 +[2025-04-24 15:49:59] (step=0057675) Train Loss: 6.3262, Train Steps/Sec: 0.90 +[2025-04-24 15:50:27] (step=0057700) Train Loss: 6.3230, Train Steps/Sec: 0.90 +[2025-04-24 15:50:54] (step=0057725) Train Loss: 6.3434, Train Steps/Sec: 0.90 +[2025-04-24 15:51:22] (step=0057750) Train Loss: 6.3370, Train Steps/Sec: 0.90 +[2025-04-24 15:51:50] (step=0057775) Train Loss: 6.2952, Train Steps/Sec: 0.90 +[2025-04-24 15:52:18] (step=0057800) Train Loss: 6.3414, Train Steps/Sec: 0.90 +[2025-04-24 15:52:45] (step=0057825) Train Loss: 6.3148, Train Steps/Sec: 0.90 +[2025-04-24 15:53:13] (step=0057850) Train Loss: 6.3360, Train Steps/Sec: 0.90 +[2025-04-24 15:53:41] (step=0057875) Train Loss: 6.2990, Train Steps/Sec: 0.91 +[2025-04-24 15:54:08] (step=0057900) Train Loss: 6.3239, Train Steps/Sec: 0.90 +[2025-04-24 15:54:36] (step=0057925) Train Loss: 6.3626, Train Steps/Sec: 0.91 +[2025-04-24 15:55:04] (step=0057950) Train Loss: 6.2671, Train Steps/Sec: 0.91 +[2025-04-24 15:55:31] (step=0057975) Train Loss: 6.2929, Train Steps/Sec: 0.91 +[2025-04-24 15:55:59] (step=0058000) Train Loss: 6.3211, Train Steps/Sec: 0.90 +[2025-04-24 15:55:59] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 15:55:59] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 15:55:59] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 16:00:53] Finish Eval in 58000 steps... +[2025-04-24 16:01:14] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0058000.pt +[2025-04-24 16:01:42] (step=0058025) Train Loss: 6.3403, Train Steps/Sec: 0.07 +[2025-04-24 16:02:10] (step=0058050) Train Loss: 6.3104, Train Steps/Sec: 0.90 +[2025-04-24 16:02:37] (step=0058075) Train Loss: 6.2870, Train Steps/Sec: 0.90 +[2025-04-24 16:03:05] (step=0058100) Train Loss: 6.3427, Train Steps/Sec: 0.90 +[2025-04-24 16:03:33] (step=0058125) Train Loss: 6.3543, Train Steps/Sec: 0.90 +[2025-04-24 16:04:01] (step=0058150) Train Loss: 6.3401, Train Steps/Sec: 0.90 +[2025-04-24 16:04:28] (step=0058175) Train Loss: 6.2792, Train Steps/Sec: 0.90 +[2025-04-24 16:04:56] (step=0058200) Train Loss: 6.2634, Train Steps/Sec: 0.90 +[2025-04-24 16:05:24] (step=0058225) Train Loss: 6.2970, Train Steps/Sec: 0.90 +[2025-04-24 16:05:51] (step=0058250) Train Loss: 6.2926, Train Steps/Sec: 0.90 +[2025-04-24 16:06:19] (step=0058275) Train Loss: 6.3428, Train Steps/Sec: 0.90 +[2025-04-24 16:06:47] (step=0058300) Train Loss: 6.3034, Train Steps/Sec: 0.89 +[2025-04-24 16:07:15] (step=0058325) Train Loss: 6.3294, Train Steps/Sec: 0.90 +[2025-04-24 16:07:42] (step=0058350) Train Loss: 6.3025, Train Steps/Sec: 0.90 +[2025-04-24 16:08:10] (step=0058375) Train Loss: 6.3177, Train Steps/Sec: 0.90 +[2025-04-24 16:08:38] (step=0058400) Train Loss: 6.3318, Train Steps/Sec: 0.90 +[2025-04-24 16:09:06] (step=0058425) Train Loss: 6.3616, Train Steps/Sec: 0.90 +[2025-04-24 16:09:33] (step=0058450) Train Loss: 6.3009, Train Steps/Sec: 0.90 +[2025-04-24 16:10:01] (step=0058475) Train Loss: 6.3446, Train Steps/Sec: 0.91 +[2025-04-24 16:10:29] (step=0058500) Train Loss: 6.3366, Train Steps/Sec: 0.90 +[2025-04-24 16:10:57] (step=0058525) Train Loss: 6.2918, Train Steps/Sec: 0.90 +[2025-04-24 16:11:24] (step=0058550) Train Loss: 6.3532, Train Steps/Sec: 0.90 +[2025-04-24 16:11:52] (step=0058575) Train Loss: 6.2985, Train Steps/Sec: 0.90 +[2025-04-24 16:12:20] (step=0058600) Train Loss: 6.3054, Train Steps/Sec: 0.90 +[2025-04-24 16:12:48] (step=0058625) Train Loss: 6.3760, Train Steps/Sec: 0.91 +[2025-04-24 16:13:15] (step=0058650) Train Loss: 6.3455, Train Steps/Sec: 0.90 +[2025-04-24 16:13:43] (step=0058675) Train Loss: 6.3433, Train Steps/Sec: 0.90 +[2025-04-24 16:14:11] (step=0058700) Train Loss: 6.3444, Train Steps/Sec: 0.90 +[2025-04-24 16:14:39] (step=0058725) Train Loss: 6.3106, Train Steps/Sec: 0.90 +[2025-04-24 16:15:06] (step=0058750) Train Loss: 6.3366, Train Steps/Sec: 0.90 +[2025-04-24 16:15:34] (step=0058775) Train Loss: 6.3141, Train Steps/Sec: 0.90 +[2025-04-24 16:16:02] (step=0058800) Train Loss: 6.2889, Train Steps/Sec: 0.90 +[2025-04-24 16:16:30] (step=0058825) Train Loss: 6.3021, Train Steps/Sec: 0.90 +[2025-04-24 16:16:57] (step=0058850) Train Loss: 6.3541, Train Steps/Sec: 0.91 +[2025-04-24 16:17:25] (step=0058875) Train Loss: 6.2576, Train Steps/Sec: 0.90 +[2025-04-24 16:17:53] (step=0058900) Train Loss: 6.3094, Train Steps/Sec: 0.90 +[2025-04-24 16:18:21] (step=0058925) Train Loss: 6.3224, Train Steps/Sec: 0.90 +[2025-04-24 16:18:48] (step=0058950) Train Loss: 6.3210, Train Steps/Sec: 0.90 +[2025-04-24 16:19:16] (step=0058975) Train Loss: 6.3302, Train Steps/Sec: 0.91 +[2025-04-24 16:19:44] (step=0059000) Train Loss: 6.3626, Train Steps/Sec: 0.90 +[2025-04-24 16:20:11] (step=0059025) Train Loss: 6.3275, Train Steps/Sec: 0.91 +[2025-04-24 16:20:39] (step=0059050) Train Loss: 6.3086, Train Steps/Sec: 0.91 +[2025-04-24 16:21:07] (step=0059075) Train Loss: 6.2642, Train Steps/Sec: 0.90 +[2025-04-24 16:21:35] (step=0059100) Train Loss: 6.3883, Train Steps/Sec: 0.90 +[2025-04-24 16:22:02] (step=0059125) Train Loss: 6.3502, Train Steps/Sec: 0.90 +[2025-04-24 16:22:30] (step=0059150) Train Loss: 6.2749, Train Steps/Sec: 0.90 +[2025-04-24 16:22:58] (step=0059175) Train Loss: 6.3466, Train Steps/Sec: 0.90 +[2025-04-24 16:23:25] (step=0059200) Train Loss: 6.3476, Train Steps/Sec: 0.90 +[2025-04-24 16:23:53] (step=0059225) Train Loss: 6.2561, Train Steps/Sec: 0.90 +[2025-04-24 16:24:21] (step=0059250) Train Loss: 6.3178, Train Steps/Sec: 0.91 +[2025-04-24 16:24:48] (step=0059275) Train Loss: 6.3445, Train Steps/Sec: 0.90 +[2025-04-24 16:25:16] (step=0059300) Train Loss: 6.2846, Train Steps/Sec: 0.89 +[2025-04-24 16:25:44] (step=0059325) Train Loss: 6.3199, Train Steps/Sec: 0.91 +[2025-04-24 16:26:12] (step=0059350) Train Loss: 6.3682, Train Steps/Sec: 0.90 +[2025-04-24 16:26:39] (step=0059375) Train Loss: 6.3290, Train Steps/Sec: 0.90 +[2025-04-24 16:27:07] (step=0059400) Train Loss: 6.3257, Train Steps/Sec: 0.90 +[2025-04-24 16:27:35] (step=0059425) Train Loss: 6.3413, Train Steps/Sec: 0.90 +[2025-04-24 16:28:03] (step=0059450) Train Loss: 6.3333, Train Steps/Sec: 0.90 +[2025-04-24 16:28:30] (step=0059475) Train Loss: 6.3210, Train Steps/Sec: 0.90 +[2025-04-24 16:28:58] (step=0059500) Train Loss: 6.3114, Train Steps/Sec: 0.90 +[2025-04-24 16:29:26] (step=0059525) Train Loss: 6.3521, Train Steps/Sec: 0.90 +[2025-04-24 16:29:54] (step=0059550) Train Loss: 6.3052, Train Steps/Sec: 0.90 +[2025-04-24 16:30:21] (step=0059575) Train Loss: 6.3182, Train Steps/Sec: 0.90 +[2025-04-24 16:30:49] (step=0059600) Train Loss: 6.3138, Train Steps/Sec: 0.90 +[2025-04-24 16:31:17] (step=0059625) Train Loss: 6.2512, Train Steps/Sec: 0.91 +[2025-04-24 16:31:44] (step=0059650) Train Loss: 6.2948, Train Steps/Sec: 0.90 +[2025-04-24 16:32:12] (step=0059675) Train Loss: 6.3044, Train Steps/Sec: 0.90 +[2025-04-24 16:32:40] (step=0059700) Train Loss: 6.3236, Train Steps/Sec: 0.90 +[2025-04-24 16:33:08] (step=0059725) Train Loss: 6.3367, Train Steps/Sec: 0.91 +[2025-04-24 16:33:35] (step=0059750) Train Loss: 6.3604, Train Steps/Sec: 0.90 +[2025-04-24 16:34:03] (step=0059775) Train Loss: 6.3144, Train Steps/Sec: 0.90 +[2025-04-24 16:34:31] (step=0059800) Train Loss: 6.3532, Train Steps/Sec: 0.90 +[2025-04-24 16:34:59] (step=0059825) Train Loss: 6.3236, Train Steps/Sec: 0.90 +[2025-04-24 16:35:26] (step=0059850) Train Loss: 6.3287, Train Steps/Sec: 0.90 +[2025-04-24 16:35:54] (step=0059875) Train Loss: 6.3425, Train Steps/Sec: 0.90 +[2025-04-24 16:36:22] (step=0059900) Train Loss: 6.2772, Train Steps/Sec: 0.90 +[2025-04-24 16:36:49] (step=0059925) Train Loss: 6.3409, Train Steps/Sec: 0.91 +[2025-04-24 16:37:17] (step=0059950) Train Loss: 6.3139, Train Steps/Sec: 0.90 +[2025-04-24 16:37:45] (step=0059975) Train Loss: 6.3816, Train Steps/Sec: 0.91 +[2025-04-24 16:38:13] (step=0060000) Train Loss: 6.2806, Train Steps/Sec: 0.90 +[2025-04-24 16:38:13] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 16:38:13] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 16:38:13] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 16:43:00] Finish Eval in 60000 steps... +[2025-04-24 16:43:21] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0060000.pt +[2025-04-24 16:43:23] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0058000.pt +[2025-04-24 16:43:52] (step=0060025) Train Loss: 6.3503, Train Steps/Sec: 0.07 +[2025-04-24 16:44:19] (step=0060050) Train Loss: 6.3163, Train Steps/Sec: 0.90 +[2025-04-24 16:44:47] (step=0060075) Train Loss: 6.3044, Train Steps/Sec: 0.90 +[2025-04-24 16:45:15] (step=0060100) Train Loss: 6.3417, Train Steps/Sec: 0.90 +[2025-04-24 16:45:43] (step=0060125) Train Loss: 6.3221, Train Steps/Sec: 0.90 +[2025-04-24 16:46:10] (step=0060150) Train Loss: 6.3354, Train Steps/Sec: 0.90 +[2025-04-24 16:46:45] (step=0060175) Train Loss: 6.2917, Train Steps/Sec: 0.72 +[2025-04-24 16:47:13] (step=0060200) Train Loss: 6.3356, Train Steps/Sec: 0.90 +[2025-04-24 16:47:41] (step=0060225) Train Loss: 6.2847, Train Steps/Sec: 0.90 +[2025-04-24 16:48:08] (step=0060250) Train Loss: 6.3315, Train Steps/Sec: 0.90 +[2025-04-24 16:48:36] (step=0060275) Train Loss: 6.3455, Train Steps/Sec: 0.90 +[2025-04-24 16:49:04] (step=0060300) Train Loss: 6.3324, Train Steps/Sec: 0.90 +[2025-04-24 16:49:32] (step=0060325) Train Loss: 6.3229, Train Steps/Sec: 0.90 +[2025-04-24 16:49:59] (step=0060350) Train Loss: 6.3650, Train Steps/Sec: 0.91 +[2025-04-24 16:50:27] (step=0060375) Train Loss: 6.3150, Train Steps/Sec: 0.90 +[2025-04-24 16:50:55] (step=0060400) Train Loss: 6.2967, Train Steps/Sec: 0.90 +[2025-04-24 16:51:22] (step=0060425) Train Loss: 6.3192, Train Steps/Sec: 0.90 +[2025-04-24 16:51:50] (step=0060450) Train Loss: 6.2796, Train Steps/Sec: 0.91 +[2025-04-24 16:52:18] (step=0060475) Train Loss: 6.3173, Train Steps/Sec: 0.90 +[2025-04-24 16:52:46] (step=0060500) Train Loss: 6.3102, Train Steps/Sec: 0.90 +[2025-04-24 16:53:13] (step=0060525) Train Loss: 6.3404, Train Steps/Sec: 0.90 +[2025-04-24 16:53:41] (step=0060550) Train Loss: 6.3216, Train Steps/Sec: 0.90 +[2025-04-24 16:54:15] (step=0060575) Train Loss: 6.3043, Train Steps/Sec: 0.73 +[2025-04-24 16:54:43] (step=0060600) Train Loss: 6.3224, Train Steps/Sec: 0.90 +[2025-04-24 16:55:11] (step=0060625) Train Loss: 6.3313, Train Steps/Sec: 0.90 +[2025-04-24 16:55:38] (step=0060650) Train Loss: 6.2895, Train Steps/Sec: 0.91 +[2025-04-24 16:56:06] (step=0060675) Train Loss: 6.3532, Train Steps/Sec: 0.91 +[2025-04-24 16:56:41] (step=0060700) Train Loss: 6.3282, Train Steps/Sec: 0.72 +[2025-04-24 16:57:09] (step=0060725) Train Loss: 6.3371, Train Steps/Sec: 0.90 +[2025-04-24 16:57:43] (step=0060750) Train Loss: 6.3280, Train Steps/Sec: 0.73 +[2025-04-24 16:58:11] (step=0060775) Train Loss: 6.3117, Train Steps/Sec: 0.90 +[2025-04-24 16:58:38] (step=0060800) Train Loss: 6.2928, Train Steps/Sec: 0.90 +[2025-04-24 16:59:06] (step=0060825) Train Loss: 6.2850, Train Steps/Sec: 0.90 +[2025-04-24 16:59:34] (step=0060850) Train Loss: 6.3255, Train Steps/Sec: 0.91 +[2025-04-24 17:00:08] (step=0060875) Train Loss: 6.2885, Train Steps/Sec: 0.73 +[2025-04-24 17:00:36] (step=0060900) Train Loss: 6.3442, Train Steps/Sec: 0.89 +[2025-04-24 17:01:09] (step=0060925) Train Loss: 6.3028, Train Steps/Sec: 0.76 +[2025-04-24 17:01:37] (step=0060950) Train Loss: 6.3116, Train Steps/Sec: 0.90 +[2025-04-24 17:02:05] (step=0060975) Train Loss: 6.2935, Train Steps/Sec: 0.90 +[2025-04-24 17:02:32] (step=0061000) Train Loss: 6.3121, Train Steps/Sec: 0.90 +[2025-04-24 17:03:00] (step=0061025) Train Loss: 6.3369, Train Steps/Sec: 0.90 +[2025-04-24 17:03:34] (step=0061050) Train Loss: 6.3160, Train Steps/Sec: 0.75 +[2025-04-24 17:04:01] (step=0061075) Train Loss: 6.3162, Train Steps/Sec: 0.90 +[2025-04-24 17:04:29] (step=0061100) Train Loss: 6.2995, Train Steps/Sec: 0.90 +[2025-04-24 17:04:57] (step=0061125) Train Loss: 6.2939, Train Steps/Sec: 0.90 +[2025-04-24 17:05:25] (step=0061150) Train Loss: 6.3304, Train Steps/Sec: 0.90 +[2025-04-24 17:05:53] (step=0061175) Train Loss: 6.3099, Train Steps/Sec: 0.90 +[2025-04-24 17:06:21] (step=0061200) Train Loss: 6.3784, Train Steps/Sec: 0.89 +[2025-04-24 17:06:49] (step=0061225) Train Loss: 6.3749, Train Steps/Sec: 0.90 +[2025-04-24 17:07:16] (step=0061250) Train Loss: 6.3284, Train Steps/Sec: 0.90 +[2025-04-24 17:07:44] (step=0061275) Train Loss: 6.3310, Train Steps/Sec: 0.90 +[2025-04-24 17:08:12] (step=0061300) Train Loss: 6.3179, Train Steps/Sec: 0.90 +[2025-04-24 17:08:40] (step=0061325) Train Loss: 6.3433, Train Steps/Sec: 0.90 +[2025-04-24 17:09:08] (step=0061350) Train Loss: 6.3213, Train Steps/Sec: 0.90 +[2025-04-24 17:09:35] (step=0061375) Train Loss: 6.3531, Train Steps/Sec: 0.90 +[2025-04-24 17:10:03] (step=0061400) Train Loss: 6.3109, Train Steps/Sec: 0.90 +[2025-04-24 17:10:31] (step=0061425) Train Loss: 6.3233, Train Steps/Sec: 0.90 +[2025-04-24 17:10:59] (step=0061450) Train Loss: 6.3188, Train Steps/Sec: 0.90 +[2025-04-24 17:11:27] (step=0061475) Train Loss: 6.3384, Train Steps/Sec: 0.90 +[2025-04-24 17:11:55] (step=0061500) Train Loss: 6.3512, Train Steps/Sec: 0.89 +[2025-04-24 17:12:23] (step=0061525) Train Loss: 6.3021, Train Steps/Sec: 0.89 +[2025-04-24 17:12:50] (step=0061550) Train Loss: 6.2827, Train Steps/Sec: 0.90 +[2025-04-24 17:13:18] (step=0061575) Train Loss: 6.2964, Train Steps/Sec: 0.90 +[2025-04-24 17:13:46] (step=0061600) Train Loss: 6.3260, Train Steps/Sec: 0.90 +[2025-04-24 17:14:14] (step=0061625) Train Loss: 6.2853, Train Steps/Sec: 0.90 +[2025-04-24 17:14:42] (step=0061650) Train Loss: 6.3327, Train Steps/Sec: 0.90 +[2025-04-24 17:15:10] (step=0061675) Train Loss: 6.3412, Train Steps/Sec: 0.90 +[2025-04-24 17:15:38] (step=0061700) Train Loss: 6.3378, Train Steps/Sec: 0.89 +[2025-04-24 17:16:05] (step=0061725) Train Loss: 6.3189, Train Steps/Sec: 0.90 +[2025-04-24 17:16:33] (step=0061750) Train Loss: 6.2872, Train Steps/Sec: 0.90 +[2025-04-24 17:17:01] (step=0061775) Train Loss: 6.2915, Train Steps/Sec: 0.90 +[2025-04-24 17:17:29] (step=0061800) Train Loss: 6.3449, Train Steps/Sec: 0.90 +[2025-04-24 17:17:57] (step=0061825) Train Loss: 6.3186, Train Steps/Sec: 0.90 +[2025-04-24 17:18:25] (step=0061850) Train Loss: 6.3165, Train Steps/Sec: 0.90 +[2025-04-24 17:18:52] (step=0061875) Train Loss: 6.2633, Train Steps/Sec: 0.90 +[2025-04-24 17:19:20] (step=0061900) Train Loss: 6.3440, Train Steps/Sec: 0.90 +[2025-04-24 17:19:48] (step=0061925) Train Loss: 6.2830, Train Steps/Sec: 0.90 +[2025-04-24 17:20:16] (step=0061950) Train Loss: 6.2966, Train Steps/Sec: 0.90 +[2025-04-24 17:20:44] (step=0061975) Train Loss: 6.2768, Train Steps/Sec: 0.90 +[2025-04-24 17:21:12] (step=0062000) Train Loss: 6.3538, Train Steps/Sec: 0.89 +[2025-04-24 17:21:12] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 17:21:12] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 17:21:12] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 17:25:59] Finish Eval in 62000 steps... +[2025-04-24 17:26:21] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0062000.pt +[2025-04-24 17:26:23] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0060000.pt +[2025-04-24 17:26:51] (step=0062025) Train Loss: 6.3304, Train Steps/Sec: 0.07 +[2025-04-24 17:27:19] (step=0062050) Train Loss: 6.3144, Train Steps/Sec: 0.90 +[2025-04-24 17:27:47] (step=0062075) Train Loss: 6.3079, Train Steps/Sec: 0.90 +[2025-04-24 17:28:15] (step=0062100) Train Loss: 6.3468, Train Steps/Sec: 0.90 +[2025-04-24 17:28:42] (step=0062125) Train Loss: 6.2781, Train Steps/Sec: 0.90 +[2025-04-24 17:29:10] (step=0062150) Train Loss: 6.3238, Train Steps/Sec: 0.90 +[2025-04-24 17:29:38] (step=0062175) Train Loss: 6.3436, Train Steps/Sec: 0.90 +[2025-04-24 17:30:06] (step=0062200) Train Loss: 6.3202, Train Steps/Sec: 0.89 +[2025-04-24 17:30:34] (step=0062225) Train Loss: 6.2816, Train Steps/Sec: 0.90 +[2025-04-24 17:31:01] (step=0062250) Train Loss: 6.2967, Train Steps/Sec: 0.90 +[2025-04-24 17:31:29] (step=0062275) Train Loss: 6.3058, Train Steps/Sec: 0.90 +[2025-04-24 17:31:57] (step=0062300) Train Loss: 6.3351, Train Steps/Sec: 0.90 +[2025-04-24 17:32:25] (step=0062325) Train Loss: 6.2873, Train Steps/Sec: 0.90 +[2025-04-24 17:32:53] (step=0062350) Train Loss: 6.3109, Train Steps/Sec: 0.90 +[2025-04-24 17:33:20] (step=0062375) Train Loss: 6.3090, Train Steps/Sec: 0.90 +[2025-04-24 17:33:48] (step=0062400) Train Loss: 6.3421, Train Steps/Sec: 0.90 +[2025-04-24 17:34:16] (step=0062425) Train Loss: 6.2883, Train Steps/Sec: 0.90 +[2025-04-24 17:34:44] (step=0062450) Train Loss: 6.3172, Train Steps/Sec: 0.90 +[2025-04-24 17:35:12] (step=0062475) Train Loss: 6.3300, Train Steps/Sec: 0.90 +[2025-04-24 17:35:39] (step=0062500) Train Loss: 6.2917, Train Steps/Sec: 0.90 +[2025-04-24 17:36:07] (step=0062525) Train Loss: 6.3215, Train Steps/Sec: 0.90 +[2025-04-24 17:36:35] (step=0062550) Train Loss: 6.3159, Train Steps/Sec: 0.90 +[2025-04-24 17:37:02] (step=0062575) Train Loss: 6.3100, Train Steps/Sec: 0.90 +[2025-04-24 17:37:30] (step=0062600) Train Loss: 6.3172, Train Steps/Sec: 0.90 +[2025-04-24 17:37:58] (step=0062625) Train Loss: 6.3015, Train Steps/Sec: 0.90 +[2025-04-24 17:38:26] (step=0062650) Train Loss: 6.3335, Train Steps/Sec: 0.90 +[2025-04-24 17:38:53] (step=0062675) Train Loss: 6.3127, Train Steps/Sec: 0.90 +[2025-04-24 17:39:21] (step=0062700) Train Loss: 6.3248, Train Steps/Sec: 0.90 +[2025-04-24 17:39:49] (step=0062725) Train Loss: 6.3022, Train Steps/Sec: 0.91 +[2025-04-24 17:40:16] (step=0062750) Train Loss: 6.3731, Train Steps/Sec: 0.91 +[2025-04-24 17:40:44] (step=0062775) Train Loss: 6.3288, Train Steps/Sec: 0.91 +[2025-04-24 17:41:12] (step=0062800) Train Loss: 6.3443, Train Steps/Sec: 0.90 +[2025-04-24 17:41:40] (step=0062825) Train Loss: 6.3224, Train Steps/Sec: 0.90 +[2025-04-24 17:42:07] (step=0062850) Train Loss: 6.3075, Train Steps/Sec: 0.90 +[2025-04-24 17:42:35] (step=0062875) Train Loss: 6.3111, Train Steps/Sec: 0.90 +[2025-04-24 17:43:03] (step=0062900) Train Loss: 6.3215, Train Steps/Sec: 0.90 +[2025-04-24 17:43:31] (step=0062925) Train Loss: 6.3229, Train Steps/Sec: 0.90 +[2025-04-24 17:43:58] (step=0062950) Train Loss: 6.3613, Train Steps/Sec: 0.90 +[2025-04-24 17:44:26] (step=0062975) Train Loss: 6.2888, Train Steps/Sec: 0.90 +[2025-04-24 17:44:54] (step=0063000) Train Loss: 6.3330, Train Steps/Sec: 0.90 +[2025-04-24 17:45:22] (step=0063025) Train Loss: 6.3043, Train Steps/Sec: 0.91 +[2025-04-24 17:45:49] (step=0063050) Train Loss: 6.3200, Train Steps/Sec: 0.90 +[2025-04-24 17:46:17] (step=0063075) Train Loss: 6.2422, Train Steps/Sec: 0.90 +[2025-04-24 17:46:45] (step=0063100) Train Loss: 6.3217, Train Steps/Sec: 0.90 +[2025-04-24 17:47:13] (step=0063125) Train Loss: 6.3217, Train Steps/Sec: 0.90 +[2025-04-24 17:47:40] (step=0063150) Train Loss: 6.3534, Train Steps/Sec: 0.90 +[2025-04-24 17:48:08] (step=0063175) Train Loss: 6.2822, Train Steps/Sec: 0.90 +[2025-04-24 17:48:36] (step=0063200) Train Loss: 6.3012, Train Steps/Sec: 0.90 +[2025-04-24 17:49:04] (step=0063225) Train Loss: 6.2771, Train Steps/Sec: 0.91 +[2025-04-24 17:49:31] (step=0063250) Train Loss: 6.3590, Train Steps/Sec: 0.90 +[2025-04-24 17:49:59] (step=0063275) Train Loss: 6.3455, Train Steps/Sec: 0.90 +[2025-04-24 17:50:27] (step=0063300) Train Loss: 6.3276, Train Steps/Sec: 0.90 +[2025-04-24 17:50:55] (step=0063325) Train Loss: 6.3164, Train Steps/Sec: 0.90 +[2025-04-24 17:51:22] (step=0063350) Train Loss: 6.3235, Train Steps/Sec: 0.90 +[2025-04-24 17:51:50] (step=0063375) Train Loss: 6.3216, Train Steps/Sec: 0.90 +[2025-04-24 17:52:18] (step=0063400) Train Loss: 6.2960, Train Steps/Sec: 0.90 +[2025-04-24 17:52:45] (step=0063425) Train Loss: 6.3230, Train Steps/Sec: 0.91 +[2025-04-24 17:53:13] (step=0063450) Train Loss: 6.3139, Train Steps/Sec: 0.90 +[2025-04-24 17:53:41] (step=0063475) Train Loss: 6.2889, Train Steps/Sec: 0.90 +[2025-04-24 17:54:09] (step=0063500) Train Loss: 6.2896, Train Steps/Sec: 0.90 +[2025-04-24 17:54:37] (step=0063525) Train Loss: 6.3383, Train Steps/Sec: 0.90 +[2025-04-24 17:55:04] (step=0063550) Train Loss: 6.3114, Train Steps/Sec: 0.90 +[2025-04-24 17:55:32] (step=0063575) Train Loss: 6.3272, Train Steps/Sec: 0.91 +[2025-04-24 17:56:00] (step=0063600) Train Loss: 6.3170, Train Steps/Sec: 0.90 +[2025-04-24 17:56:27] (step=0063625) Train Loss: 6.3246, Train Steps/Sec: 0.90 +[2025-04-24 17:56:55] (step=0063650) Train Loss: 6.2969, Train Steps/Sec: 0.90 +[2025-04-24 17:57:23] (step=0063675) Train Loss: 6.3062, Train Steps/Sec: 0.90 +[2025-04-24 17:57:51] (step=0063700) Train Loss: 6.3326, Train Steps/Sec: 0.90 +[2025-04-24 17:58:19] (step=0063725) Train Loss: 6.3590, Train Steps/Sec: 0.90 +[2025-04-24 17:58:46] (step=0063750) Train Loss: 6.3301, Train Steps/Sec: 0.90 +[2025-04-24 17:59:14] (step=0063775) Train Loss: 6.2811, Train Steps/Sec: 0.90 +[2025-04-24 17:59:42] (step=0063800) Train Loss: 6.3075, Train Steps/Sec: 0.89 +[2025-04-24 18:00:10] (step=0063825) Train Loss: 6.3050, Train Steps/Sec: 0.90 +[2025-04-24 18:00:38] (step=0063850) Train Loss: 6.2993, Train Steps/Sec: 0.90 +[2025-04-24 18:01:05] (step=0063875) Train Loss: 6.3161, Train Steps/Sec: 0.90 +[2025-04-24 18:01:33] (step=0063900) Train Loss: 6.2872, Train Steps/Sec: 0.90 +[2025-04-24 18:02:01] (step=0063925) Train Loss: 6.3456, Train Steps/Sec: 0.90 +[2025-04-24 18:02:29] (step=0063950) Train Loss: 6.3406, Train Steps/Sec: 0.90 +[2025-04-24 18:02:57] (step=0063975) Train Loss: 6.2883, Train Steps/Sec: 0.90 +[2025-04-24 18:03:25] (step=0064000) Train Loss: 6.3845, Train Steps/Sec: 0.90 +[2025-04-24 18:03:25] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 18:03:25] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 18:03:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 18:08:11] Finish Eval in 64000 steps... +[2025-04-24 18:08:33] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0064000.pt +[2025-04-24 18:08:35] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0062000.pt +[2025-04-24 18:09:03] (step=0064025) Train Loss: 6.2839, Train Steps/Sec: 0.07 +[2025-04-24 18:09:30] (step=0064050) Train Loss: 6.3318, Train Steps/Sec: 0.90 +[2025-04-24 18:09:58] (step=0064075) Train Loss: 6.3614, Train Steps/Sec: 0.90 +[2025-04-24 18:10:26] (step=0064100) Train Loss: 6.3052, Train Steps/Sec: 0.90 +[2025-04-24 18:10:54] (step=0064125) Train Loss: 6.3590, Train Steps/Sec: 0.90 +[2025-04-24 18:11:21] (step=0064150) Train Loss: 6.3499, Train Steps/Sec: 0.90 +[2025-04-24 18:11:49] (step=0064175) Train Loss: 6.3079, Train Steps/Sec: 0.90 +[2025-04-24 18:12:17] (step=0064200) Train Loss: 6.3007, Train Steps/Sec: 0.90 +[2025-04-24 18:12:45] (step=0064225) Train Loss: 6.3449, Train Steps/Sec: 0.90 +[2025-04-24 18:13:13] (step=0064250) Train Loss: 6.2880, Train Steps/Sec: 0.90 +[2025-04-24 18:13:40] (step=0064275) Train Loss: 6.3027, Train Steps/Sec: 0.90 +[2025-04-24 18:14:08] (step=0064300) Train Loss: 6.2975, Train Steps/Sec: 0.90 +[2025-04-24 18:14:36] (step=0064325) Train Loss: 6.3129, Train Steps/Sec: 0.90 +[2025-04-24 18:15:04] (step=0064350) Train Loss: 6.2814, Train Steps/Sec: 0.90 +[2025-04-24 18:15:31] (step=0064375) Train Loss: 6.3423, Train Steps/Sec: 0.90 +[2025-04-24 18:15:59] (step=0064400) Train Loss: 6.3141, Train Steps/Sec: 0.90 +[2025-04-24 18:16:27] (step=0064425) Train Loss: 6.3216, Train Steps/Sec: 0.90 +[2025-04-24 18:16:55] (step=0064450) Train Loss: 6.3007, Train Steps/Sec: 0.90 +[2025-04-24 18:17:22] (step=0064475) Train Loss: 6.3726, Train Steps/Sec: 0.90 +[2025-04-24 18:17:50] (step=0064500) Train Loss: 6.2956, Train Steps/Sec: 0.90 +[2025-04-24 18:18:18] (step=0064525) Train Loss: 6.3156, Train Steps/Sec: 0.90 +[2025-04-24 18:18:46] (step=0064550) Train Loss: 6.3656, Train Steps/Sec: 0.90 +[2025-04-24 18:19:13] (step=0064575) Train Loss: 6.2682, Train Steps/Sec: 0.90 +[2025-04-24 18:19:41] (step=0064600) Train Loss: 6.3432, Train Steps/Sec: 0.90 +[2025-04-24 18:20:09] (step=0064625) Train Loss: 6.2993, Train Steps/Sec: 0.90 +[2025-04-24 18:20:37] (step=0064650) Train Loss: 6.3281, Train Steps/Sec: 0.90 +[2025-04-24 18:21:04] (step=0064675) Train Loss: 6.3156, Train Steps/Sec: 0.90 +[2025-04-24 18:21:32] (step=0064700) Train Loss: 6.3169, Train Steps/Sec: 0.90 +[2025-04-24 18:22:00] (step=0064725) Train Loss: 6.3343, Train Steps/Sec: 0.90 +[2025-04-24 18:22:28] (step=0064750) Train Loss: 6.3509, Train Steps/Sec: 0.90 +[2025-04-24 18:22:55] (step=0064775) Train Loss: 6.3073, Train Steps/Sec: 0.91 +[2025-04-24 18:23:23] (step=0064800) Train Loss: 6.2816, Train Steps/Sec: 0.90 +[2025-04-24 18:23:51] (step=0064825) Train Loss: 6.3353, Train Steps/Sec: 0.90 +[2025-04-24 18:24:19] (step=0064850) Train Loss: 6.3568, Train Steps/Sec: 0.90 +[2025-04-24 18:24:46] (step=0064875) Train Loss: 6.2991, Train Steps/Sec: 0.90 +[2025-04-24 18:25:14] (step=0064900) Train Loss: 6.2545, Train Steps/Sec: 0.90 +[2025-04-24 18:25:42] (step=0064925) Train Loss: 6.3167, Train Steps/Sec: 0.91 +[2025-04-24 18:26:09] (step=0064950) Train Loss: 6.2872, Train Steps/Sec: 0.90 +[2025-04-24 18:26:37] (step=0064975) Train Loss: 6.3841, Train Steps/Sec: 0.90 +[2025-04-24 18:27:05] (step=0065000) Train Loss: 6.2724, Train Steps/Sec: 0.90 +[2025-04-24 18:27:33] (step=0065025) Train Loss: 6.3125, Train Steps/Sec: 0.90 +[2025-04-24 18:28:06] (step=0065050) Train Loss: 6.3150, Train Steps/Sec: 0.76 +[2025-04-24 18:28:34] (step=0065075) Train Loss: 6.3393, Train Steps/Sec: 0.90 +[2025-04-24 18:29:02] (step=0065100) Train Loss: 6.3575, Train Steps/Sec: 0.90 +[2025-04-24 18:29:30] (step=0065125) Train Loss: 6.3128, Train Steps/Sec: 0.90 +[2025-04-24 18:29:57] (step=0065150) Train Loss: 6.3311, Train Steps/Sec: 0.90 +[2025-04-24 18:30:25] (step=0065175) Train Loss: 6.3588, Train Steps/Sec: 0.90 +[2025-04-24 18:30:53] (step=0065200) Train Loss: 6.3312, Train Steps/Sec: 0.90 +[2025-04-24 18:31:21] (step=0065225) Train Loss: 6.3264, Train Steps/Sec: 0.90 +[2025-04-24 18:31:48] (step=0065250) Train Loss: 6.2899, Train Steps/Sec: 0.90 +[2025-04-24 18:32:16] (step=0065275) Train Loss: 6.3122, Train Steps/Sec: 0.90 +[2025-04-24 18:32:44] (step=0065300) Train Loss: 6.3075, Train Steps/Sec: 0.90 +[2025-04-24 18:33:11] (step=0065325) Train Loss: 6.3360, Train Steps/Sec: 0.90 +[2025-04-24 18:33:39] (step=0065350) Train Loss: 6.2785, Train Steps/Sec: 0.91 +[2025-04-24 18:34:07] (step=0065375) Train Loss: 6.3380, Train Steps/Sec: 0.90 +[2025-04-24 18:34:35] (step=0065400) Train Loss: 6.3230, Train Steps/Sec: 0.90 +[2025-04-24 18:35:02] (step=0065425) Train Loss: 6.3219, Train Steps/Sec: 0.90 +[2025-04-24 18:35:37] (step=0065450) Train Loss: 6.2963, Train Steps/Sec: 0.73 +[2025-04-24 18:36:04] (step=0065475) Train Loss: 6.2665, Train Steps/Sec: 0.91 +[2025-04-24 18:36:32] (step=0065500) Train Loss: 6.3055, Train Steps/Sec: 0.90 +[2025-04-24 18:37:00] (step=0065525) Train Loss: 6.2850, Train Steps/Sec: 0.90 +[2025-04-24 18:37:28] (step=0065550) Train Loss: 6.3445, Train Steps/Sec: 0.90 +[2025-04-24 18:37:55] (step=0065575) Train Loss: 6.2991, Train Steps/Sec: 0.90 +[2025-04-24 18:38:29] (step=0065600) Train Loss: 6.2821, Train Steps/Sec: 0.75 +[2025-04-24 18:39:01] (step=0065625) Train Loss: 6.3374, Train Steps/Sec: 0.77 +[2025-04-24 18:39:29] (step=0065650) Train Loss: 6.2943, Train Steps/Sec: 0.90 +[2025-04-24 18:39:57] (step=0065675) Train Loss: 6.3175, Train Steps/Sec: 0.91 +[2025-04-24 18:40:30] (step=0065700) Train Loss: 6.2695, Train Steps/Sec: 0.76 +[2025-04-24 18:41:03] (step=0065725) Train Loss: 6.3218, Train Steps/Sec: 0.76 +[2025-04-24 18:41:30] (step=0065750) Train Loss: 6.2976, Train Steps/Sec: 0.90 +[2025-04-24 18:41:58] (step=0065775) Train Loss: 6.2601, Train Steps/Sec: 0.90 +[2025-04-24 18:42:26] (step=0065800) Train Loss: 6.2912, Train Steps/Sec: 0.90 +[2025-04-24 18:42:54] (step=0065825) Train Loss: 6.3002, Train Steps/Sec: 0.90 +[2025-04-24 18:43:21] (step=0065850) Train Loss: 6.3101, Train Steps/Sec: 0.90 +[2025-04-24 18:43:49] (step=0065875) Train Loss: 6.3175, Train Steps/Sec: 0.90 +[2025-04-24 18:44:17] (step=0065900) Train Loss: 6.2887, Train Steps/Sec: 0.90 +[2025-04-24 18:44:45] (step=0065925) Train Loss: 6.3643, Train Steps/Sec: 0.90 +[2025-04-24 18:45:12] (step=0065950) Train Loss: 6.2727, Train Steps/Sec: 0.90 +[2025-04-24 18:45:46] (step=0065975) Train Loss: 6.3126, Train Steps/Sec: 0.74 +[2025-04-24 18:46:14] (step=0066000) Train Loss: 6.2573, Train Steps/Sec: 0.90 +[2025-04-24 18:46:14] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 18:46:14] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 18:46:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 18:51:01] Finish Eval in 66000 steps... +[2025-04-24 18:51:23] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0066000.pt +[2025-04-24 18:51:25] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0064000.pt +[2025-04-24 18:51:53] (step=0066025) Train Loss: 6.3237, Train Steps/Sec: 0.07 +[2025-04-24 18:52:20] (step=0066050) Train Loss: 6.3104, Train Steps/Sec: 0.91 +[2025-04-24 18:52:48] (step=0066075) Train Loss: 6.3216, Train Steps/Sec: 0.90 +[2025-04-24 18:53:16] (step=0066100) Train Loss: 6.3364, Train Steps/Sec: 0.90 +[2025-04-24 18:53:44] (step=0066125) Train Loss: 6.2455, Train Steps/Sec: 0.91 +[2025-04-24 18:54:11] (step=0066150) Train Loss: 6.2264, Train Steps/Sec: 0.90 +[2025-04-24 18:54:39] (step=0066175) Train Loss: 6.3378, Train Steps/Sec: 0.90 +[2025-04-24 18:55:07] (step=0066200) Train Loss: 6.3190, Train Steps/Sec: 0.90 +[2025-04-24 18:55:35] (step=0066225) Train Loss: 6.3046, Train Steps/Sec: 0.90 +[2025-04-24 18:56:02] (step=0066250) Train Loss: 6.3438, Train Steps/Sec: 0.91 +[2025-04-24 18:56:30] (step=0066275) Train Loss: 6.3496, Train Steps/Sec: 0.90 +[2025-04-24 18:56:58] (step=0066300) Train Loss: 6.3372, Train Steps/Sec: 0.90 +[2025-04-24 18:57:26] (step=0066325) Train Loss: 6.3116, Train Steps/Sec: 0.90 +[2025-04-24 18:57:53] (step=0066350) Train Loss: 6.3663, Train Steps/Sec: 0.90 +[2025-04-24 18:58:21] (step=0066375) Train Loss: 6.3475, Train Steps/Sec: 0.90 +[2025-04-24 18:58:49] (step=0066400) Train Loss: 6.3010, Train Steps/Sec: 0.89 +[2025-04-24 18:59:17] (step=0066425) Train Loss: 6.3169, Train Steps/Sec: 0.90 +[2025-04-24 18:59:44] (step=0066450) Train Loss: 6.2998, Train Steps/Sec: 0.90 +[2025-04-24 19:00:12] (step=0066475) Train Loss: 6.2896, Train Steps/Sec: 0.89 +[2025-04-24 19:00:40] (step=0066500) Train Loss: 6.2768, Train Steps/Sec: 0.90 +[2025-04-24 19:01:08] (step=0066525) Train Loss: 6.2909, Train Steps/Sec: 0.90 +[2025-04-24 19:01:36] (step=0066550) Train Loss: 6.3574, Train Steps/Sec: 0.90 +[2025-04-24 19:02:03] (step=0066575) Train Loss: 6.3397, Train Steps/Sec: 0.90 +[2025-04-24 19:02:31] (step=0066600) Train Loss: 6.2963, Train Steps/Sec: 0.90 +[2025-04-24 19:02:59] (step=0066625) Train Loss: 6.2739, Train Steps/Sec: 0.90 +[2025-04-24 19:03:26] (step=0066650) Train Loss: 6.2971, Train Steps/Sec: 0.91 +[2025-04-24 19:03:54] (step=0066675) Train Loss: 6.3014, Train Steps/Sec: 0.90 +[2025-04-24 19:04:22] (step=0066700) Train Loss: 6.3006, Train Steps/Sec: 0.90 +[2025-04-24 19:04:50] (step=0066725) Train Loss: 6.2837, Train Steps/Sec: 0.90 +[2025-04-24 19:05:18] (step=0066750) Train Loss: 6.3068, Train Steps/Sec: 0.90 +[2025-04-24 19:05:45] (step=0066775) Train Loss: 6.3091, Train Steps/Sec: 0.90 +[2025-04-24 19:06:13] (step=0066800) Train Loss: 6.2785, Train Steps/Sec: 0.90 +[2025-04-24 19:06:41] (step=0066825) Train Loss: 6.2653, Train Steps/Sec: 0.90 +[2025-04-24 19:07:09] (step=0066850) Train Loss: 6.3011, Train Steps/Sec: 0.90 +[2025-04-24 19:07:36] (step=0066875) Train Loss: 6.3032, Train Steps/Sec: 0.90 +[2025-04-24 19:08:04] (step=0066900) Train Loss: 6.3323, Train Steps/Sec: 0.90 +[2025-04-24 19:08:32] (step=0066925) Train Loss: 6.3332, Train Steps/Sec: 0.90 +[2025-04-24 19:09:00] (step=0066950) Train Loss: 6.2868, Train Steps/Sec: 0.90 +[2025-04-24 19:09:27] (step=0066975) Train Loss: 6.2807, Train Steps/Sec: 0.90 +[2025-04-24 19:09:55] (step=0067000) Train Loss: 6.3213, Train Steps/Sec: 0.89 +[2025-04-24 19:10:23] (step=0067025) Train Loss: 6.3396, Train Steps/Sec: 0.90 +[2025-04-24 19:10:51] (step=0067050) Train Loss: 6.3091, Train Steps/Sec: 0.90 +[2025-04-24 19:11:18] (step=0067075) Train Loss: 6.3617, Train Steps/Sec: 0.90 +[2025-04-24 19:11:46] (step=0067100) Train Loss: 6.2808, Train Steps/Sec: 0.90 +[2025-04-24 19:12:14] (step=0067125) Train Loss: 6.3083, Train Steps/Sec: 0.90 +[2025-04-24 19:12:42] (step=0067150) Train Loss: 6.3327, Train Steps/Sec: 0.90 +[2025-04-24 19:13:09] (step=0067175) Train Loss: 6.3145, Train Steps/Sec: 0.90 +[2025-04-24 19:13:37] (step=0067200) Train Loss: 6.3285, Train Steps/Sec: 0.90 +[2025-04-24 19:14:05] (step=0067225) Train Loss: 6.3363, Train Steps/Sec: 0.91 +[2025-04-24 19:14:32] (step=0067250) Train Loss: 6.3142, Train Steps/Sec: 0.90 +[2025-04-24 19:15:00] (step=0067275) Train Loss: 6.2803, Train Steps/Sec: 0.90 +[2025-04-24 19:15:28] (step=0067300) Train Loss: 6.3404, Train Steps/Sec: 0.90 +[2025-04-24 19:15:56] (step=0067325) Train Loss: 6.2950, Train Steps/Sec: 0.90 +[2025-04-24 19:16:23] (step=0067350) Train Loss: 6.3258, Train Steps/Sec: 0.90 +[2025-04-24 19:16:51] (step=0067375) Train Loss: 6.2837, Train Steps/Sec: 0.91 +[2025-04-24 19:17:19] (step=0067400) Train Loss: 6.3646, Train Steps/Sec: 0.90 +[2025-04-24 19:17:46] (step=0067425) Train Loss: 6.3444, Train Steps/Sec: 0.90 +[2025-04-24 19:18:14] (step=0067450) Train Loss: 6.3482, Train Steps/Sec: 0.91 +[2025-04-24 19:18:42] (step=0067475) Train Loss: 6.3496, Train Steps/Sec: 0.90 +[2025-04-24 19:19:10] (step=0067500) Train Loss: 6.2976, Train Steps/Sec: 0.90 +[2025-04-24 19:19:37] (step=0067525) Train Loss: 6.3188, Train Steps/Sec: 0.91 +[2025-04-24 19:20:05] (step=0067550) Train Loss: 6.3099, Train Steps/Sec: 0.91 +[2025-04-24 19:20:33] (step=0067575) Train Loss: 6.2898, Train Steps/Sec: 0.90 +[2025-04-24 19:21:00] (step=0067600) Train Loss: 6.3043, Train Steps/Sec: 0.90 +[2025-04-24 19:21:28] (step=0067625) Train Loss: 6.3296, Train Steps/Sec: 0.90 +[2025-04-24 19:21:56] (step=0067650) Train Loss: 6.3162, Train Steps/Sec: 0.90 +[2025-04-24 19:22:23] (step=0067675) Train Loss: 6.3143, Train Steps/Sec: 0.91 +[2025-04-24 19:22:51] (step=0067700) Train Loss: 6.2603, Train Steps/Sec: 0.90 +[2025-04-24 19:23:19] (step=0067725) Train Loss: 6.2656, Train Steps/Sec: 0.91 +[2025-04-24 19:23:46] (step=0067750) Train Loss: 6.3023, Train Steps/Sec: 0.91 +[2025-04-24 19:24:14] (step=0067775) Train Loss: 6.2887, Train Steps/Sec: 0.91 +[2025-04-24 19:24:42] (step=0067800) Train Loss: 6.2518, Train Steps/Sec: 0.90 +[2025-04-24 19:25:09] (step=0067825) Train Loss: 6.2711, Train Steps/Sec: 0.91 +[2025-04-24 19:25:37] (step=0067850) Train Loss: 6.3081, Train Steps/Sec: 0.91 +[2025-04-24 19:26:04] (step=0067875) Train Loss: 6.2641, Train Steps/Sec: 0.91 +[2025-04-24 19:26:32] (step=0067900) Train Loss: 6.2905, Train Steps/Sec: 0.90 +[2025-04-24 19:27:00] (step=0067925) Train Loss: 6.2750, Train Steps/Sec: 0.90 +[2025-04-24 19:27:28] (step=0067950) Train Loss: 6.2853, Train Steps/Sec: 0.90 +[2025-04-24 19:27:55] (step=0067975) Train Loss: 6.3313, Train Steps/Sec: 0.90 +[2025-04-24 19:28:23] (step=0068000) Train Loss: 6.3291, Train Steps/Sec: 0.89 +[2025-04-24 19:28:23] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 19:28:23] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 19:28:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 19:33:08] Finish Eval in 68000 steps... +[2025-04-24 19:33:29] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0068000.pt +[2025-04-24 19:33:31] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0066000.pt +[2025-04-24 19:33:59] (step=0068025) Train Loss: 6.3361, Train Steps/Sec: 0.07 +[2025-04-24 19:34:27] (step=0068050) Train Loss: 6.3306, Train Steps/Sec: 0.90 +[2025-04-24 19:34:55] (step=0068075) Train Loss: 6.2553, Train Steps/Sec: 0.90 +[2025-04-24 19:35:22] (step=0068100) Train Loss: 6.3235, Train Steps/Sec: 0.90 +[2025-04-24 19:35:50] (step=0068125) Train Loss: 6.2964, Train Steps/Sec: 0.90 +[2025-04-24 19:36:18] (step=0068150) Train Loss: 6.3285, Train Steps/Sec: 0.90 +[2025-04-24 19:36:46] (step=0068175) Train Loss: 6.3537, Train Steps/Sec: 0.90 +[2025-04-24 19:37:13] (step=0068200) Train Loss: 6.3410, Train Steps/Sec: 0.90 +[2025-04-24 19:37:41] (step=0068225) Train Loss: 6.2892, Train Steps/Sec: 0.91 +[2025-04-24 19:38:09] (step=0068250) Train Loss: 6.2986, Train Steps/Sec: 0.90 +[2025-04-24 19:38:37] (step=0068275) Train Loss: 6.2738, Train Steps/Sec: 0.89 +[2025-04-24 19:39:05] (step=0068300) Train Loss: 6.3553, Train Steps/Sec: 0.90 +[2025-04-24 19:39:32] (step=0068325) Train Loss: 6.2820, Train Steps/Sec: 0.90 +[2025-04-24 19:40:00] (step=0068350) Train Loss: 6.3232, Train Steps/Sec: 0.90 +[2025-04-24 19:40:28] (step=0068375) Train Loss: 6.3375, Train Steps/Sec: 0.90 +[2025-04-24 19:40:56] (step=0068400) Train Loss: 6.3400, Train Steps/Sec: 0.90 +[2025-04-24 19:41:24] (step=0068425) Train Loss: 6.3508, Train Steps/Sec: 0.90 +[2025-04-24 19:41:51] (step=0068450) Train Loss: 6.2718, Train Steps/Sec: 0.90 +[2025-04-24 19:42:19] (step=0068475) Train Loss: 6.2937, Train Steps/Sec: 0.90 +[2025-04-24 19:42:47] (step=0068500) Train Loss: 6.3797, Train Steps/Sec: 0.90 +[2025-04-24 19:43:14] (step=0068525) Train Loss: 6.2684, Train Steps/Sec: 0.91 +[2025-04-24 19:43:42] (step=0068550) Train Loss: 6.3548, Train Steps/Sec: 0.90 +[2025-04-24 19:44:10] (step=0068575) Train Loss: 6.3114, Train Steps/Sec: 0.90 +[2025-04-24 19:44:38] (step=0068600) Train Loss: 6.3005, Train Steps/Sec: 0.90 +[2025-04-24 19:45:06] (step=0068625) Train Loss: 6.3250, Train Steps/Sec: 0.90 +[2025-04-24 19:45:33] (step=0068650) Train Loss: 6.2994, Train Steps/Sec: 0.91 +[2025-04-24 19:46:01] (step=0068675) Train Loss: 6.2506, Train Steps/Sec: 0.90 +[2025-04-24 19:46:29] (step=0068700) Train Loss: 6.3281, Train Steps/Sec: 0.90 +[2025-04-24 19:46:57] (step=0068725) Train Loss: 6.2756, Train Steps/Sec: 0.90 +[2025-04-24 19:47:24] (step=0068750) Train Loss: 6.3101, Train Steps/Sec: 0.91 +[2025-04-24 19:47:52] (step=0068775) Train Loss: 6.3699, Train Steps/Sec: 0.90 +[2025-04-24 19:48:20] (step=0068800) Train Loss: 6.2929, Train Steps/Sec: 0.90 +[2025-04-24 19:48:48] (step=0068825) Train Loss: 6.2840, Train Steps/Sec: 0.90 +[2025-04-24 19:49:15] (step=0068850) Train Loss: 6.3441, Train Steps/Sec: 0.91 +[2025-04-24 19:49:43] (step=0068875) Train Loss: 6.2706, Train Steps/Sec: 0.91 +[2025-04-24 19:50:11] (step=0068900) Train Loss: 6.3292, Train Steps/Sec: 0.90 +[2025-04-24 19:50:38] (step=0068925) Train Loss: 6.2568, Train Steps/Sec: 0.90 +[2025-04-24 19:51:06] (step=0068950) Train Loss: 6.3114, Train Steps/Sec: 0.90 +[2025-04-24 19:51:34] (step=0068975) Train Loss: 6.2827, Train Steps/Sec: 0.90 +[2025-04-24 19:52:02] (step=0069000) Train Loss: 6.3544, Train Steps/Sec: 0.90 +[2025-04-24 19:52:29] (step=0069025) Train Loss: 6.3281, Train Steps/Sec: 0.90 +[2025-04-24 19:52:57] (step=0069050) Train Loss: 6.3346, Train Steps/Sec: 0.90 +[2025-04-24 19:53:25] (step=0069075) Train Loss: 6.2955, Train Steps/Sec: 0.91 +[2025-04-24 19:53:53] (step=0069100) Train Loss: 6.2818, Train Steps/Sec: 0.90 +[2025-04-24 19:54:20] (step=0069125) Train Loss: 6.3512, Train Steps/Sec: 0.90 +[2025-04-24 19:54:48] (step=0069150) Train Loss: 6.3645, Train Steps/Sec: 0.90 +[2025-04-24 19:55:16] (step=0069175) Train Loss: 6.2823, Train Steps/Sec: 0.91 +[2025-04-24 19:55:43] (step=0069200) Train Loss: 6.2899, Train Steps/Sec: 0.90 +[2025-04-24 19:56:11] (step=0069225) Train Loss: 6.3161, Train Steps/Sec: 0.91 +[2025-04-24 19:56:39] (step=0069250) Train Loss: 6.3342, Train Steps/Sec: 0.90 +[2025-04-24 19:57:07] (step=0069275) Train Loss: 6.2506, Train Steps/Sec: 0.90 +[2025-04-24 19:57:35] (step=0069300) Train Loss: 6.2682, Train Steps/Sec: 0.90 +[2025-04-24 19:58:02] (step=0069325) Train Loss: 6.2497, Train Steps/Sec: 0.91 +[2025-04-24 19:58:30] (step=0069350) Train Loss: 6.2963, Train Steps/Sec: 0.90 +[2025-04-24 19:58:57] (step=0069375) Train Loss: 6.2752, Train Steps/Sec: 0.91 +[2025-04-24 19:59:25] (step=0069400) Train Loss: 6.3084, Train Steps/Sec: 0.90 +[2025-04-24 19:59:53] (step=0069425) Train Loss: 6.3071, Train Steps/Sec: 0.90 +[2025-04-24 20:00:21] (step=0069450) Train Loss: 6.3292, Train Steps/Sec: 0.91 +[2025-04-24 20:00:48] (step=0069475) Train Loss: 6.3184, Train Steps/Sec: 0.91 +[2025-04-24 20:01:16] (step=0069500) Train Loss: 6.2997, Train Steps/Sec: 0.90 +[2025-04-24 20:01:44] (step=0069525) Train Loss: 6.2633, Train Steps/Sec: 0.91 +[2025-04-24 20:02:11] (step=0069550) Train Loss: 6.2944, Train Steps/Sec: 0.90 +[2025-04-24 20:02:39] (step=0069575) Train Loss: 6.3097, Train Steps/Sec: 0.90 +[2025-04-24 20:03:07] (step=0069600) Train Loss: 6.2891, Train Steps/Sec: 0.90 +[2025-04-24 20:03:35] (step=0069625) Train Loss: 6.2859, Train Steps/Sec: 0.90 +[2025-04-24 20:04:02] (step=0069650) Train Loss: 6.3171, Train Steps/Sec: 0.90 +[2025-04-24 20:04:30] (step=0069675) Train Loss: 6.3368, Train Steps/Sec: 0.90 +[2025-04-24 20:04:58] (step=0069700) Train Loss: 6.2966, Train Steps/Sec: 0.90 +[2025-04-24 20:05:25] (step=0069725) Train Loss: 6.2988, Train Steps/Sec: 0.91 +[2025-04-24 20:05:53] (step=0069750) Train Loss: 6.2522, Train Steps/Sec: 0.91 +[2025-04-24 20:06:21] (step=0069775) Train Loss: 6.2807, Train Steps/Sec: 0.90 +[2025-04-24 20:06:48] (step=0069800) Train Loss: 6.3169, Train Steps/Sec: 0.90 +[2025-04-24 20:07:16] (step=0069825) Train Loss: 6.3289, Train Steps/Sec: 0.90 +[2025-04-24 20:07:49] (step=0069850) Train Loss: 6.2990, Train Steps/Sec: 0.77 +[2025-04-24 20:08:17] (step=0069875) Train Loss: 6.3386, Train Steps/Sec: 0.89 +[2025-04-24 20:08:45] (step=0069900) Train Loss: 6.3381, Train Steps/Sec: 0.90 +[2025-04-24 20:09:12] (step=0069925) Train Loss: 6.2071, Train Steps/Sec: 0.90 +[2025-04-24 20:09:40] (step=0069950) Train Loss: 6.3548, Train Steps/Sec: 0.90 +[2025-04-24 20:10:08] (step=0069975) Train Loss: 6.3212, Train Steps/Sec: 0.90 +[2025-04-24 20:10:35] (step=0070000) Train Loss: 6.3524, Train Steps/Sec: 0.90 +[2025-04-24 20:10:36] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 20:10:36] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 20:10:36] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 20:15:22] Finish Eval in 70000 steps... +[2025-04-24 20:15:43] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0070000.pt +[2025-04-24 20:15:45] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0068000.pt +[2025-04-24 20:16:13] (step=0070025) Train Loss: 6.3638, Train Steps/Sec: 0.07 +[2025-04-24 20:16:41] (step=0070050) Train Loss: 6.2935, Train Steps/Sec: 0.90 +[2025-04-24 20:17:08] (step=0070075) Train Loss: 6.3478, Train Steps/Sec: 0.90 +[2025-04-24 20:17:36] (step=0070100) Train Loss: 6.3158, Train Steps/Sec: 0.90 +[2025-04-24 20:18:04] (step=0070125) Train Loss: 6.2867, Train Steps/Sec: 0.90 +[2025-04-24 20:18:32] (step=0070150) Train Loss: 6.3214, Train Steps/Sec: 0.90 +[2025-04-24 20:19:00] (step=0070175) Train Loss: 6.3095, Train Steps/Sec: 0.90 +[2025-04-24 20:19:27] (step=0070200) Train Loss: 6.3033, Train Steps/Sec: 0.90 +[2025-04-24 20:20:00] (step=0070225) Train Loss: 6.3238, Train Steps/Sec: 0.76 +[2025-04-24 20:20:28] (step=0070250) Train Loss: 6.2938, Train Steps/Sec: 0.90 +[2025-04-24 20:21:01] (step=0070275) Train Loss: 6.3125, Train Steps/Sec: 0.76 +[2025-04-24 20:21:29] (step=0070300) Train Loss: 6.3208, Train Steps/Sec: 0.90 +[2025-04-24 20:21:56] (step=0070325) Train Loss: 6.3675, Train Steps/Sec: 0.90 +[2025-04-24 20:22:24] (step=0070350) Train Loss: 6.2870, Train Steps/Sec: 0.90 +[2025-04-24 20:23:03] (step=0070375) Train Loss: 6.2464, Train Steps/Sec: 0.64 +[2025-04-24 20:23:31] (step=0070400) Train Loss: 6.3510, Train Steps/Sec: 0.90 +[2025-04-24 20:24:04] (step=0070425) Train Loss: 6.3140, Train Steps/Sec: 0.76 +[2025-04-24 20:24:32] (step=0070450) Train Loss: 6.3172, Train Steps/Sec: 0.90 +[2025-04-24 20:24:59] (step=0070475) Train Loss: 6.3038, Train Steps/Sec: 0.90 +[2025-04-24 20:25:27] (step=0070500) Train Loss: 6.3081, Train Steps/Sec: 0.90 +[2025-04-24 20:25:55] (step=0070525) Train Loss: 6.3045, Train Steps/Sec: 0.90 +[2025-04-24 20:26:23] (step=0070550) Train Loss: 6.2952, Train Steps/Sec: 0.90 +[2025-04-24 20:26:51] (step=0070575) Train Loss: 6.2746, Train Steps/Sec: 0.90 +[2025-04-24 20:27:18] (step=0070600) Train Loss: 6.2969, Train Steps/Sec: 0.90 +[2025-04-24 20:27:46] (step=0070625) Train Loss: 6.3603, Train Steps/Sec: 0.90 +[2025-04-24 20:28:14] (step=0070650) Train Loss: 6.2986, Train Steps/Sec: 0.90 +[2025-04-24 20:28:41] (step=0070675) Train Loss: 6.2894, Train Steps/Sec: 0.91 +[2025-04-24 20:29:09] (step=0070700) Train Loss: 6.2988, Train Steps/Sec: 0.89 +[2025-04-24 20:29:37] (step=0070725) Train Loss: 6.2767, Train Steps/Sec: 0.90 +[2025-04-24 20:30:05] (step=0070750) Train Loss: 6.3087, Train Steps/Sec: 0.90 +[2025-04-24 20:30:33] (step=0070775) Train Loss: 6.3322, Train Steps/Sec: 0.90 +[2025-04-24 20:31:00] (step=0070800) Train Loss: 6.3074, Train Steps/Sec: 0.90 +[2025-04-24 20:31:28] (step=0070825) Train Loss: 6.2802, Train Steps/Sec: 0.90 +[2025-04-24 20:31:56] (step=0070850) Train Loss: 6.3167, Train Steps/Sec: 0.90 +[2025-04-24 20:32:24] (step=0070875) Train Loss: 6.2784, Train Steps/Sec: 0.90 +[2025-04-24 20:32:51] (step=0070900) Train Loss: 6.3193, Train Steps/Sec: 0.90 +[2025-04-24 20:33:19] (step=0070925) Train Loss: 6.2956, Train Steps/Sec: 0.90 +[2025-04-24 20:33:47] (step=0070950) Train Loss: 6.2763, Train Steps/Sec: 0.90 +[2025-04-24 20:34:15] (step=0070975) Train Loss: 6.3284, Train Steps/Sec: 0.91 +[2025-04-24 20:34:42] (step=0071000) Train Loss: 6.3265, Train Steps/Sec: 0.90 +[2025-04-24 20:35:10] (step=0071025) Train Loss: 6.3116, Train Steps/Sec: 0.90 +[2025-04-24 20:35:38] (step=0071050) Train Loss: 6.3067, Train Steps/Sec: 0.90 +[2025-04-24 20:36:06] (step=0071075) Train Loss: 6.3440, Train Steps/Sec: 0.90 +[2025-04-24 20:36:33] (step=0071100) Train Loss: 6.3288, Train Steps/Sec: 0.90 +[2025-04-24 20:37:01] (step=0071125) Train Loss: 6.3157, Train Steps/Sec: 0.90 +[2025-04-24 20:37:29] (step=0071150) Train Loss: 6.3131, Train Steps/Sec: 0.90 +[2025-04-24 20:37:57] (step=0071175) Train Loss: 6.2783, Train Steps/Sec: 0.90 +[2025-04-24 20:38:30] (step=0071200) Train Loss: 6.2539, Train Steps/Sec: 0.75 +[2025-04-24 20:38:58] (step=0071225) Train Loss: 6.2633, Train Steps/Sec: 0.91 +[2025-04-24 20:39:25] (step=0071250) Train Loss: 6.3146, Train Steps/Sec: 0.90 +[2025-04-24 20:39:53] (step=0071275) Train Loss: 6.3180, Train Steps/Sec: 0.90 +[2025-04-24 20:40:21] (step=0071300) Train Loss: 6.2932, Train Steps/Sec: 0.90 +[2025-04-24 20:40:49] (step=0071325) Train Loss: 6.3309, Train Steps/Sec: 0.90 +[2025-04-24 20:41:16] (step=0071350) Train Loss: 6.3025, Train Steps/Sec: 0.91 +[2025-04-24 20:41:44] (step=0071375) Train Loss: 6.2968, Train Steps/Sec: 0.90 +[2025-04-24 20:42:12] (step=0071400) Train Loss: 6.3173, Train Steps/Sec: 0.90 +[2025-04-24 20:42:40] (step=0071425) Train Loss: 6.3232, Train Steps/Sec: 0.90 +[2025-04-24 20:43:07] (step=0071450) Train Loss: 6.2974, Train Steps/Sec: 0.91 +[2025-04-24 20:43:35] (step=0071475) Train Loss: 6.2991, Train Steps/Sec: 0.91 +[2025-04-24 20:44:03] (step=0071500) Train Loss: 6.2467, Train Steps/Sec: 0.90 +[2025-04-24 20:44:30] (step=0071525) Train Loss: 6.3446, Train Steps/Sec: 0.90 +[2025-04-24 20:44:58] (step=0071550) Train Loss: 6.2539, Train Steps/Sec: 0.91 +[2025-04-24 20:45:26] (step=0071575) Train Loss: 6.3225, Train Steps/Sec: 0.90 +[2025-04-24 20:45:53] (step=0071600) Train Loss: 6.2962, Train Steps/Sec: 0.90 +[2025-04-24 20:46:21] (step=0071625) Train Loss: 6.2930, Train Steps/Sec: 0.90 +[2025-04-24 20:46:49] (step=0071650) Train Loss: 6.3070, Train Steps/Sec: 0.90 +[2025-04-24 20:47:16] (step=0071675) Train Loss: 6.2677, Train Steps/Sec: 0.91 +[2025-04-24 20:47:44] (step=0071700) Train Loss: 6.2733, Train Steps/Sec: 0.90 +[2025-04-24 20:48:12] (step=0071725) Train Loss: 6.2963, Train Steps/Sec: 0.90 +[2025-04-24 20:48:40] (step=0071750) Train Loss: 6.3088, Train Steps/Sec: 0.90 +[2025-04-24 20:49:07] (step=0071775) Train Loss: 6.2860, Train Steps/Sec: 0.90 +[2025-04-24 20:49:35] (step=0071800) Train Loss: 6.3005, Train Steps/Sec: 0.90 +[2025-04-24 20:50:03] (step=0071825) Train Loss: 6.2473, Train Steps/Sec: 0.90 +[2025-04-24 20:50:31] (step=0071850) Train Loss: 6.3002, Train Steps/Sec: 0.90 +[2025-04-24 20:50:58] (step=0071875) Train Loss: 6.3358, Train Steps/Sec: 0.90 +[2025-04-24 20:51:26] (step=0071900) Train Loss: 6.3282, Train Steps/Sec: 0.90 +[2025-04-24 20:51:54] (step=0071925) Train Loss: 6.2850, Train Steps/Sec: 0.90 +[2025-04-24 20:52:21] (step=0071950) Train Loss: 6.3039, Train Steps/Sec: 0.90 +[2025-04-24 20:52:49] (step=0071975) Train Loss: 6.3195, Train Steps/Sec: 0.90 +[2025-04-24 20:53:17] (step=0072000) Train Loss: 6.3039, Train Steps/Sec: 0.90 +[2025-04-24 20:53:17] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 20:53:17] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 20:53:17] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 20:58:01] Finish Eval in 72000 steps... +[2025-04-24 20:58:21] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0072000.pt +[2025-04-24 20:58:23] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0070000.pt +[2025-04-24 20:58:52] (step=0072025) Train Loss: 6.2627, Train Steps/Sec: 0.07 +[2025-04-24 20:59:19] (step=0072050) Train Loss: 6.2974, Train Steps/Sec: 0.90 +[2025-04-24 20:59:47] (step=0072075) Train Loss: 6.3281, Train Steps/Sec: 0.90 +[2025-04-24 21:00:15] (step=0072100) Train Loss: 6.2769, Train Steps/Sec: 0.90 +[2025-04-24 21:00:43] (step=0072125) Train Loss: 6.3154, Train Steps/Sec: 0.90 +[2025-04-24 21:01:10] (step=0072150) Train Loss: 6.2704, Train Steps/Sec: 0.90 +[2025-04-24 21:01:38] (step=0072175) Train Loss: 6.3267, Train Steps/Sec: 0.90 +[2025-04-24 21:02:06] (step=0072200) Train Loss: 6.3470, Train Steps/Sec: 0.90 +[2025-04-24 21:02:33] (step=0072225) Train Loss: 6.3171, Train Steps/Sec: 0.91 +[2025-04-24 21:03:01] (step=0072250) Train Loss: 6.2998, Train Steps/Sec: 0.90 +[2025-04-24 21:03:29] (step=0072275) Train Loss: 6.2902, Train Steps/Sec: 0.90 +[2025-04-24 21:03:57] (step=0072300) Train Loss: 6.2798, Train Steps/Sec: 0.90 +[2025-04-24 21:04:25] (step=0072325) Train Loss: 6.3313, Train Steps/Sec: 0.90 +[2025-04-24 21:04:52] (step=0072350) Train Loss: 6.2533, Train Steps/Sec: 0.90 +[2025-04-24 21:05:20] (step=0072375) Train Loss: 6.2766, Train Steps/Sec: 0.90 +[2025-04-24 21:05:48] (step=0072400) Train Loss: 6.2788, Train Steps/Sec: 0.90 +[2025-04-24 21:06:16] (step=0072425) Train Loss: 6.3500, Train Steps/Sec: 0.90 +[2025-04-24 21:06:43] (step=0072450) Train Loss: 6.3715, Train Steps/Sec: 0.90 +[2025-04-24 21:07:11] (step=0072475) Train Loss: 6.3105, Train Steps/Sec: 0.90 +[2025-04-24 21:07:39] (step=0072500) Train Loss: 6.2876, Train Steps/Sec: 0.90 +[2025-04-24 21:08:07] (step=0072525) Train Loss: 6.3012, Train Steps/Sec: 0.90 +[2025-04-24 21:08:34] (step=0072550) Train Loss: 6.3021, Train Steps/Sec: 0.90 +[2025-04-24 21:09:02] (step=0072575) Train Loss: 6.3326, Train Steps/Sec: 0.90 +[2025-04-24 21:09:30] (step=0072600) Train Loss: 6.2812, Train Steps/Sec: 0.90 +[2025-04-24 21:09:57] (step=0072625) Train Loss: 6.3193, Train Steps/Sec: 0.90 +[2025-04-24 21:10:25] (step=0072650) Train Loss: 6.3219, Train Steps/Sec: 0.90 +[2025-04-24 21:10:53] (step=0072675) Train Loss: 6.2810, Train Steps/Sec: 0.90 +[2025-04-24 21:11:21] (step=0072700) Train Loss: 6.2577, Train Steps/Sec: 0.90 +[2025-04-24 21:11:49] (step=0072725) Train Loss: 6.3139, Train Steps/Sec: 0.90 +[2025-04-24 21:12:16] (step=0072750) Train Loss: 6.2752, Train Steps/Sec: 0.90 +[2025-04-24 21:12:44] (step=0072775) Train Loss: 6.3029, Train Steps/Sec: 0.90 +[2025-04-24 21:13:12] (step=0072800) Train Loss: 6.3039, Train Steps/Sec: 0.90 +[2025-04-24 21:13:40] (step=0072825) Train Loss: 6.3105, Train Steps/Sec: 0.90 +[2025-04-24 21:14:07] (step=0072850) Train Loss: 6.3014, Train Steps/Sec: 0.90 +[2025-04-24 21:14:35] (step=0072875) Train Loss: 6.2858, Train Steps/Sec: 0.91 +[2025-04-24 21:15:03] (step=0072900) Train Loss: 6.3278, Train Steps/Sec: 0.89 +[2025-04-24 21:15:31] (step=0072925) Train Loss: 6.2768, Train Steps/Sec: 0.90 +[2025-04-24 21:15:58] (step=0072950) Train Loss: 6.3274, Train Steps/Sec: 0.90 +[2025-04-24 21:16:26] (step=0072975) Train Loss: 6.3171, Train Steps/Sec: 0.90 +[2025-04-24 21:16:54] (step=0073000) Train Loss: 6.3085, Train Steps/Sec: 0.90 +[2025-04-24 21:17:22] (step=0073025) Train Loss: 6.2625, Train Steps/Sec: 0.90 +[2025-04-24 21:17:50] (step=0073050) Train Loss: 6.3074, Train Steps/Sec: 0.90 +[2025-04-24 21:18:17] (step=0073075) Train Loss: 6.3343, Train Steps/Sec: 0.90 +[2025-04-24 21:18:45] (step=0073100) Train Loss: 6.3196, Train Steps/Sec: 0.90 +[2025-04-24 21:19:13] (step=0073125) Train Loss: 6.3235, Train Steps/Sec: 0.90 +[2025-04-24 21:19:41] (step=0073150) Train Loss: 6.2531, Train Steps/Sec: 0.90 +[2025-04-24 21:20:08] (step=0073175) Train Loss: 6.3130, Train Steps/Sec: 0.90 +[2025-04-24 21:20:36] (step=0073200) Train Loss: 6.2940, Train Steps/Sec: 0.90 +[2025-04-24 21:21:04] (step=0073225) Train Loss: 6.3163, Train Steps/Sec: 0.90 +[2025-04-24 21:21:32] (step=0073250) Train Loss: 6.2478, Train Steps/Sec: 0.91 +[2025-04-24 21:21:59] (step=0073275) Train Loss: 6.3184, Train Steps/Sec: 0.90 +[2025-04-24 21:22:27] (step=0073300) Train Loss: 6.2868, Train Steps/Sec: 0.90 +[2025-04-24 21:22:55] (step=0073325) Train Loss: 6.2726, Train Steps/Sec: 0.91 +[2025-04-24 21:23:22] (step=0073350) Train Loss: 6.2554, Train Steps/Sec: 0.91 +[2025-04-24 21:23:50] (step=0073375) Train Loss: 6.3001, Train Steps/Sec: 0.90 +[2025-04-24 21:24:18] (step=0073400) Train Loss: 6.3416, Train Steps/Sec: 0.90 +[2025-04-24 21:24:45] (step=0073425) Train Loss: 6.3121, Train Steps/Sec: 0.90 +[2025-04-24 21:25:13] (step=0073450) Train Loss: 6.2953, Train Steps/Sec: 0.91 +[2025-04-24 21:25:41] (step=0073475) Train Loss: 6.3075, Train Steps/Sec: 0.90 +[2025-04-24 21:26:09] (step=0073500) Train Loss: 6.2918, Train Steps/Sec: 0.90 +[2025-04-24 21:26:36] (step=0073525) Train Loss: 6.3229, Train Steps/Sec: 0.90 +[2025-04-24 21:27:04] (step=0073550) Train Loss: 6.2982, Train Steps/Sec: 0.91 +[2025-04-24 21:27:32] (step=0073575) Train Loss: 6.2901, Train Steps/Sec: 0.90 +[2025-04-24 21:28:00] (step=0073600) Train Loss: 6.2934, Train Steps/Sec: 0.89 +[2025-04-24 21:28:27] (step=0073625) Train Loss: 6.3271, Train Steps/Sec: 0.90 +[2025-04-24 21:28:55] (step=0073650) Train Loss: 6.2632, Train Steps/Sec: 0.91 +[2025-04-24 21:29:23] (step=0073675) Train Loss: 6.2859, Train Steps/Sec: 0.90 +[2025-04-24 21:29:51] (step=0073700) Train Loss: 6.3038, Train Steps/Sec: 0.90 +[2025-04-24 21:30:18] (step=0073725) Train Loss: 6.2971, Train Steps/Sec: 0.90 +[2025-04-24 21:30:46] (step=0073750) Train Loss: 6.2687, Train Steps/Sec: 0.90 +[2025-04-24 21:31:14] (step=0073775) Train Loss: 6.3468, Train Steps/Sec: 0.90 +[2025-04-24 21:31:42] (step=0073800) Train Loss: 6.2747, Train Steps/Sec: 0.90 +[2025-04-24 21:32:09] (step=0073825) Train Loss: 6.3135, Train Steps/Sec: 0.90 +[2025-04-24 21:32:37] (step=0073850) Train Loss: 6.3306, Train Steps/Sec: 0.91 +[2025-04-24 21:33:05] (step=0073875) Train Loss: 6.3177, Train Steps/Sec: 0.90 +[2025-04-24 21:33:32] (step=0073900) Train Loss: 6.3511, Train Steps/Sec: 0.90 +[2025-04-24 21:34:00] (step=0073925) Train Loss: 6.2958, Train Steps/Sec: 0.90 +[2025-04-24 21:34:28] (step=0073950) Train Loss: 6.2872, Train Steps/Sec: 0.90 +[2025-04-24 21:34:56] (step=0073975) Train Loss: 6.3096, Train Steps/Sec: 0.90 +[2025-04-24 21:35:24] (step=0074000) Train Loss: 6.2681, Train Steps/Sec: 0.90 +[2025-04-24 21:35:24] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 21:35:24] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 21:35:24] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 21:40:12] Finish Eval in 74000 steps... +[2025-04-24 21:40:33] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0074000.pt +[2025-04-24 21:40:35] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0072000.pt +[2025-04-24 21:41:03] (step=0074025) Train Loss: 6.2745, Train Steps/Sec: 0.07 +[2025-04-24 21:41:31] (step=0074050) Train Loss: 6.2923, Train Steps/Sec: 0.90 +[2025-04-24 21:41:59] (step=0074075) Train Loss: 6.2699, Train Steps/Sec: 0.90 +[2025-04-24 21:42:27] (step=0074100) Train Loss: 6.2758, Train Steps/Sec: 0.90 +[2025-04-24 21:42:54] (step=0074125) Train Loss: 6.2748, Train Steps/Sec: 0.90 +[2025-04-24 21:43:22] (step=0074150) Train Loss: 6.3117, Train Steps/Sec: 0.90 +[2025-04-24 21:43:50] (step=0074175) Train Loss: 6.3042, Train Steps/Sec: 0.90 +[2025-04-24 21:44:17] (step=0074200) Train Loss: 6.3236, Train Steps/Sec: 0.90 +[2025-04-24 21:44:45] (step=0074225) Train Loss: 6.3315, Train Steps/Sec: 0.90 +[2025-04-24 21:45:13] (step=0074250) Train Loss: 6.3021, Train Steps/Sec: 0.90 +[2025-04-24 21:45:40] (step=0074275) Train Loss: 6.2806, Train Steps/Sec: 0.91 +[2025-04-24 21:46:08] (step=0074300) Train Loss: 6.2484, Train Steps/Sec: 0.90 +[2025-04-24 21:46:36] (step=0074325) Train Loss: 6.3228, Train Steps/Sec: 0.90 +[2025-04-24 21:47:04] (step=0074350) Train Loss: 6.3701, Train Steps/Sec: 0.90 +[2025-04-24 21:47:31] (step=0074375) Train Loss: 6.3521, Train Steps/Sec: 0.90 +[2025-04-24 21:47:59] (step=0074400) Train Loss: 6.3185, Train Steps/Sec: 0.90 +[2025-04-24 21:48:27] (step=0074425) Train Loss: 6.3603, Train Steps/Sec: 0.90 +[2025-04-24 21:48:55] (step=0074450) Train Loss: 6.3421, Train Steps/Sec: 0.91 +[2025-04-24 21:49:22] (step=0074475) Train Loss: 6.2940, Train Steps/Sec: 0.91 +[2025-04-24 21:49:50] (step=0074500) Train Loss: 6.3286, Train Steps/Sec: 0.89 +[2025-04-24 21:50:18] (step=0074525) Train Loss: 6.3292, Train Steps/Sec: 0.90 +[2025-04-24 21:50:46] (step=0074550) Train Loss: 6.3545, Train Steps/Sec: 0.90 +[2025-04-24 21:51:13] (step=0074575) Train Loss: 6.3021, Train Steps/Sec: 0.90 +[2025-04-24 21:51:41] (step=0074600) Train Loss: 6.3218, Train Steps/Sec: 0.90 +[2025-04-24 21:52:14] (step=0074625) Train Loss: 6.3184, Train Steps/Sec: 0.77 +[2025-04-24 21:52:42] (step=0074650) Train Loss: 6.3253, Train Steps/Sec: 0.90 +[2025-04-24 21:53:09] (step=0074675) Train Loss: 6.3496, Train Steps/Sec: 0.90 +[2025-04-24 21:53:37] (step=0074700) Train Loss: 6.3238, Train Steps/Sec: 0.90 +[2025-04-24 21:54:05] (step=0074725) Train Loss: 6.3160, Train Steps/Sec: 0.90 +[2025-04-24 21:54:33] (step=0074750) Train Loss: 6.3521, Train Steps/Sec: 0.90 +[2025-04-24 21:55:00] (step=0074775) Train Loss: 6.3766, Train Steps/Sec: 0.90 +[2025-04-24 21:55:28] (step=0074800) Train Loss: 6.3149, Train Steps/Sec: 0.90 +[2025-04-24 21:55:56] (step=0074825) Train Loss: 6.3149, Train Steps/Sec: 0.91 +[2025-04-24 21:56:23] (step=0074850) Train Loss: 6.2692, Train Steps/Sec: 0.90 +[2025-04-24 21:56:51] (step=0074875) Train Loss: 6.2832, Train Steps/Sec: 0.90 +[2025-04-24 21:57:19] (step=0074900) Train Loss: 6.3569, Train Steps/Sec: 0.90 +[2025-04-24 21:57:47] (step=0074925) Train Loss: 6.3231, Train Steps/Sec: 0.90 +[2025-04-24 21:58:19] (step=0074950) Train Loss: 6.3269, Train Steps/Sec: 0.77 +[2025-04-24 21:58:47] (step=0074975) Train Loss: 6.3047, Train Steps/Sec: 0.90 +[2025-04-24 21:59:15] (step=0075000) Train Loss: 6.3085, Train Steps/Sec: 0.90 +[2025-04-24 21:59:47] (step=0075025) Train Loss: 6.3017, Train Steps/Sec: 0.76 +[2025-04-24 22:00:20] (step=0075050) Train Loss: 6.2947, Train Steps/Sec: 0.77 +[2025-04-24 22:00:48] (step=0075075) Train Loss: 6.2741, Train Steps/Sec: 0.90 +[2025-04-24 22:01:16] (step=0075100) Train Loss: 6.3193, Train Steps/Sec: 0.90 +[2025-04-24 22:01:49] (step=0075125) Train Loss: 6.3241, Train Steps/Sec: 0.76 +[2025-04-24 22:02:22] (step=0075150) Train Loss: 6.3013, Train Steps/Sec: 0.77 +[2025-04-24 22:02:49] (step=0075175) Train Loss: 6.3260, Train Steps/Sec: 0.90 +[2025-04-24 22:03:17] (step=0075200) Train Loss: 6.3278, Train Steps/Sec: 0.90 +[2025-04-24 22:03:45] (step=0075225) Train Loss: 6.2289, Train Steps/Sec: 0.90 +[2025-04-24 22:04:13] (step=0075250) Train Loss: 6.3501, Train Steps/Sec: 0.90 +[2025-04-24 22:04:40] (step=0075275) Train Loss: 6.3070, Train Steps/Sec: 0.90 +[2025-04-24 22:05:08] (step=0075300) Train Loss: 6.2892, Train Steps/Sec: 0.90 +[2025-04-24 22:05:36] (step=0075325) Train Loss: 6.3169, Train Steps/Sec: 0.90 +[2025-04-24 22:06:03] (step=0075350) Train Loss: 6.3049, Train Steps/Sec: 0.90 +[2025-04-24 22:06:31] (step=0075375) Train Loss: 6.2868, Train Steps/Sec: 0.90 +[2025-04-24 22:06:59] (step=0075400) Train Loss: 6.3019, Train Steps/Sec: 0.90 +[2025-04-24 22:07:27] (step=0075425) Train Loss: 6.3012, Train Steps/Sec: 0.90 +[2025-04-24 22:07:55] (step=0075450) Train Loss: 6.2838, Train Steps/Sec: 0.90 +[2025-04-24 22:08:22] (step=0075475) Train Loss: 6.3376, Train Steps/Sec: 0.90 +[2025-04-24 22:08:50] (step=0075500) Train Loss: 6.2986, Train Steps/Sec: 0.90 +[2025-04-24 22:09:18] (step=0075525) Train Loss: 6.2431, Train Steps/Sec: 0.91 +[2025-04-24 22:09:46] (step=0075550) Train Loss: 6.3022, Train Steps/Sec: 0.90 +[2025-04-24 22:10:13] (step=0075575) Train Loss: 6.2903, Train Steps/Sec: 0.90 +[2025-04-24 22:10:41] (step=0075600) Train Loss: 6.3237, Train Steps/Sec: 0.90 +[2025-04-24 22:11:09] (step=0075625) Train Loss: 6.3362, Train Steps/Sec: 0.91 +[2025-04-24 22:11:37] (step=0075650) Train Loss: 6.2655, Train Steps/Sec: 0.90 +[2025-04-24 22:12:04] (step=0075675) Train Loss: 6.3133, Train Steps/Sec: 0.90 +[2025-04-24 22:12:32] (step=0075700) Train Loss: 6.2951, Train Steps/Sec: 0.90 +[2025-04-24 22:13:00] (step=0075725) Train Loss: 6.3471, Train Steps/Sec: 0.90 +[2025-04-24 22:13:28] (step=0075750) Train Loss: 6.3075, Train Steps/Sec: 0.90 +[2025-04-24 22:13:55] (step=0075775) Train Loss: 6.3028, Train Steps/Sec: 0.90 +[2025-04-24 22:14:23] (step=0075800) Train Loss: 6.2136, Train Steps/Sec: 0.90 +[2025-04-24 22:14:51] (step=0075825) Train Loss: 6.3236, Train Steps/Sec: 0.90 +[2025-04-24 22:15:18] (step=0075850) Train Loss: 6.2577, Train Steps/Sec: 0.90 +[2025-04-24 22:15:46] (step=0075875) Train Loss: 6.3237, Train Steps/Sec: 0.90 +[2025-04-24 22:16:14] (step=0075900) Train Loss: 6.3072, Train Steps/Sec: 0.90 +[2025-04-24 22:16:42] (step=0075925) Train Loss: 6.3252, Train Steps/Sec: 0.90 +[2025-04-24 22:17:09] (step=0075950) Train Loss: 6.3051, Train Steps/Sec: 0.90 +[2025-04-24 22:17:37] (step=0075975) Train Loss: 6.3279, Train Steps/Sec: 0.90 +[2025-04-24 22:18:05] (step=0076000) Train Loss: 6.3243, Train Steps/Sec: 0.90 +[2025-04-24 22:18:05] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 22:18:05] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 22:18:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 22:22:50] Finish Eval in 76000 steps... +[2025-04-24 22:23:12] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0076000.pt +[2025-04-24 22:23:14] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0074000.pt +[2025-04-24 22:23:43] (step=0076025) Train Loss: 6.2960, Train Steps/Sec: 0.07 +[2025-04-24 22:24:10] (step=0076050) Train Loss: 6.3248, Train Steps/Sec: 0.90 +[2025-04-24 22:24:38] (step=0076075) Train Loss: 6.2920, Train Steps/Sec: 0.90 +[2025-04-24 22:25:06] (step=0076100) Train Loss: 6.3132, Train Steps/Sec: 0.90 +[2025-04-24 22:25:34] (step=0076125) Train Loss: 6.3547, Train Steps/Sec: 0.90 +[2025-04-24 22:26:01] (step=0076150) Train Loss: 6.2921, Train Steps/Sec: 0.90 +[2025-04-24 22:26:29] (step=0076175) Train Loss: 6.3275, Train Steps/Sec: 0.90 +[2025-04-24 22:26:57] (step=0076200) Train Loss: 6.2448, Train Steps/Sec: 0.90 +[2025-04-24 22:27:25] (step=0076225) Train Loss: 6.2372, Train Steps/Sec: 0.91 +[2025-04-24 22:27:52] (step=0076250) Train Loss: 6.3004, Train Steps/Sec: 0.91 +[2025-04-24 22:28:20] (step=0076275) Train Loss: 6.3131, Train Steps/Sec: 0.90 +[2025-04-24 22:28:48] (step=0076300) Train Loss: 6.3087, Train Steps/Sec: 0.90 +[2025-04-24 22:29:15] (step=0076325) Train Loss: 6.2633, Train Steps/Sec: 0.90 +[2025-04-24 22:29:43] (step=0076350) Train Loss: 6.2984, Train Steps/Sec: 0.91 +[2025-04-24 22:30:17] (step=0076375) Train Loss: 6.3410, Train Steps/Sec: 0.74 +[2025-04-24 22:30:44] (step=0076400) Train Loss: 6.2952, Train Steps/Sec: 0.90 +[2025-04-24 22:31:12] (step=0076425) Train Loss: 6.2965, Train Steps/Sec: 0.90 +[2025-04-24 22:31:40] (step=0076450) Train Loss: 6.2975, Train Steps/Sec: 0.91 +[2025-04-24 22:32:07] (step=0076475) Train Loss: 6.2784, Train Steps/Sec: 0.90 +[2025-04-24 22:32:35] (step=0076500) Train Loss: 6.3102, Train Steps/Sec: 0.90 +[2025-04-24 22:33:03] (step=0076525) Train Loss: 6.3320, Train Steps/Sec: 0.90 +[2025-04-24 22:33:31] (step=0076550) Train Loss: 6.3202, Train Steps/Sec: 0.90 +[2025-04-24 22:33:59] (step=0076575) Train Loss: 6.2958, Train Steps/Sec: 0.90 +[2025-04-24 22:34:26] (step=0076600) Train Loss: 6.2984, Train Steps/Sec: 0.90 +[2025-04-24 22:34:54] (step=0076625) Train Loss: 6.2667, Train Steps/Sec: 0.90 +[2025-04-24 22:35:22] (step=0076650) Train Loss: 6.3031, Train Steps/Sec: 0.90 +[2025-04-24 22:35:50] (step=0076675) Train Loss: 6.2950, Train Steps/Sec: 0.90 +[2025-04-24 22:36:18] (step=0076700) Train Loss: 6.3268, Train Steps/Sec: 0.90 +[2025-04-24 22:36:45] (step=0076725) Train Loss: 6.3045, Train Steps/Sec: 0.90 +[2025-04-24 22:37:13] (step=0076750) Train Loss: 6.3179, Train Steps/Sec: 0.91 +[2025-04-24 22:37:40] (step=0076775) Train Loss: 6.2889, Train Steps/Sec: 0.90 +[2025-04-24 22:38:08] (step=0076800) Train Loss: 6.3053, Train Steps/Sec: 0.90 +[2025-04-24 22:38:36] (step=0076825) Train Loss: 6.2841, Train Steps/Sec: 0.91 +[2025-04-24 22:39:03] (step=0076850) Train Loss: 6.3237, Train Steps/Sec: 0.91 +[2025-04-24 22:39:31] (step=0076875) Train Loss: 6.2881, Train Steps/Sec: 0.90 +[2025-04-24 22:39:59] (step=0076900) Train Loss: 6.2906, Train Steps/Sec: 0.90 +[2025-04-24 22:40:27] (step=0076925) Train Loss: 6.3433, Train Steps/Sec: 0.90 +[2025-04-24 22:40:54] (step=0076950) Train Loss: 6.2805, Train Steps/Sec: 0.91 +[2025-04-24 22:41:22] (step=0076975) Train Loss: 6.3019, Train Steps/Sec: 0.90 +[2025-04-24 22:41:50] (step=0077000) Train Loss: 6.2659, Train Steps/Sec: 0.90 +[2025-04-24 22:42:17] (step=0077025) Train Loss: 6.2931, Train Steps/Sec: 0.90 +[2025-04-24 22:42:45] (step=0077050) Train Loss: 6.3218, Train Steps/Sec: 0.90 +[2025-04-24 22:43:13] (step=0077075) Train Loss: 6.2964, Train Steps/Sec: 0.91 +[2025-04-24 22:43:41] (step=0077100) Train Loss: 6.3347, Train Steps/Sec: 0.90 +[2025-04-24 22:44:08] (step=0077125) Train Loss: 6.3067, Train Steps/Sec: 0.90 +[2025-04-24 22:44:36] (step=0077150) Train Loss: 6.3069, Train Steps/Sec: 0.90 +[2025-04-24 22:45:04] (step=0077175) Train Loss: 6.2616, Train Steps/Sec: 0.90 +[2025-04-24 22:45:31] (step=0077200) Train Loss: 6.3413, Train Steps/Sec: 0.90 +[2025-04-24 22:45:59] (step=0077225) Train Loss: 6.2545, Train Steps/Sec: 0.90 +[2025-04-24 22:46:27] (step=0077250) Train Loss: 6.3122, Train Steps/Sec: 0.91 +[2025-04-24 22:46:54] (step=0077275) Train Loss: 6.3178, Train Steps/Sec: 0.90 +[2025-04-24 22:47:22] (step=0077300) Train Loss: 6.2742, Train Steps/Sec: 0.90 +[2025-04-24 22:47:50] (step=0077325) Train Loss: 6.3125, Train Steps/Sec: 0.90 +[2025-04-24 22:48:18] (step=0077350) Train Loss: 6.2814, Train Steps/Sec: 0.90 +[2025-04-24 22:48:45] (step=0077375) Train Loss: 6.3218, Train Steps/Sec: 0.91 +[2025-04-24 22:49:13] (step=0077400) Train Loss: 6.3238, Train Steps/Sec: 0.90 +[2025-04-24 22:49:41] (step=0077425) Train Loss: 6.2766, Train Steps/Sec: 0.90 +[2025-04-24 22:50:09] (step=0077450) Train Loss: 6.3266, Train Steps/Sec: 0.90 +[2025-04-24 22:50:36] (step=0077475) Train Loss: 6.3077, Train Steps/Sec: 0.90 +[2025-04-24 22:51:04] (step=0077500) Train Loss: 6.3222, Train Steps/Sec: 0.90 +[2025-04-24 22:51:32] (step=0077525) Train Loss: 6.2936, Train Steps/Sec: 0.90 +[2025-04-24 22:52:00] (step=0077550) Train Loss: 6.3118, Train Steps/Sec: 0.90 +[2025-04-24 22:52:27] (step=0077575) Train Loss: 6.3296, Train Steps/Sec: 0.90 +[2025-04-24 22:52:55] (step=0077600) Train Loss: 6.2579, Train Steps/Sec: 0.90 +[2025-04-24 22:53:23] (step=0077625) Train Loss: 6.2781, Train Steps/Sec: 0.90 +[2025-04-24 22:53:51] (step=0077650) Train Loss: 6.2629, Train Steps/Sec: 0.90 +[2025-04-24 22:54:18] (step=0077675) Train Loss: 6.2349, Train Steps/Sec: 0.91 +[2025-04-24 22:54:46] (step=0077700) Train Loss: 6.3132, Train Steps/Sec: 0.90 +[2025-04-24 22:55:14] (step=0077725) Train Loss: 6.3177, Train Steps/Sec: 0.90 +[2025-04-24 22:55:42] (step=0077750) Train Loss: 6.2808, Train Steps/Sec: 0.90 +[2025-04-24 22:56:09] (step=0077775) Train Loss: 6.3115, Train Steps/Sec: 0.90 +[2025-04-24 22:56:37] (step=0077800) Train Loss: 6.3009, Train Steps/Sec: 0.90 +[2025-04-24 22:57:05] (step=0077825) Train Loss: 6.2597, Train Steps/Sec: 0.90 +[2025-04-24 22:57:33] (step=0077850) Train Loss: 6.2705, Train Steps/Sec: 0.90 +[2025-04-24 22:58:00] (step=0077875) Train Loss: 6.2567, Train Steps/Sec: 0.90 +[2025-04-24 22:58:28] (step=0077900) Train Loss: 6.3009, Train Steps/Sec: 0.90 +[2025-04-24 22:58:56] (step=0077925) Train Loss: 6.3172, Train Steps/Sec: 0.91 +[2025-04-24 22:59:23] (step=0077950) Train Loss: 6.2977, Train Steps/Sec: 0.90 +[2025-04-24 22:59:51] (step=0077975) Train Loss: 6.3201, Train Steps/Sec: 0.90 +[2025-04-24 23:00:19] (step=0078000) Train Loss: 6.3455, Train Steps/Sec: 0.90 +[2025-04-24 23:00:19] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 23:00:19] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 23:00:19] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 23:05:04] Finish Eval in 78000 steps... +[2025-04-24 23:05:25] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0078000.pt +[2025-04-24 23:05:27] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0076000.pt +[2025-04-24 23:05:55] (step=0078025) Train Loss: 6.2432, Train Steps/Sec: 0.07 +[2025-04-24 23:06:23] (step=0078050) Train Loss: 6.2832, Train Steps/Sec: 0.90 +[2025-04-24 23:06:51] (step=0078075) Train Loss: 6.3157, Train Steps/Sec: 0.90 +[2025-04-24 23:07:19] (step=0078100) Train Loss: 6.2525, Train Steps/Sec: 0.90 +[2025-04-24 23:07:46] (step=0078125) Train Loss: 6.3142, Train Steps/Sec: 0.90 +[2025-04-24 23:08:14] (step=0078150) Train Loss: 6.2946, Train Steps/Sec: 0.91 +[2025-04-24 23:08:41] (step=0078175) Train Loss: 6.2980, Train Steps/Sec: 0.91 +[2025-04-24 23:09:09] (step=0078200) Train Loss: 6.3420, Train Steps/Sec: 0.90 +[2025-04-24 23:09:37] (step=0078225) Train Loss: 6.3538, Train Steps/Sec: 0.89 +[2025-04-24 23:10:05] (step=0078250) Train Loss: 6.3438, Train Steps/Sec: 0.90 +[2025-04-24 23:10:33] (step=0078275) Train Loss: 6.3317, Train Steps/Sec: 0.90 +[2025-04-24 23:11:01] (step=0078300) Train Loss: 6.3116, Train Steps/Sec: 0.90 +[2025-04-24 23:11:28] (step=0078325) Train Loss: 6.2943, Train Steps/Sec: 0.90 +[2025-04-24 23:11:56] (step=0078350) Train Loss: 6.2936, Train Steps/Sec: 0.90 +[2025-04-24 23:12:24] (step=0078375) Train Loss: 6.3334, Train Steps/Sec: 0.90 +[2025-04-24 23:12:51] (step=0078400) Train Loss: 6.3046, Train Steps/Sec: 0.90 +[2025-04-24 23:13:19] (step=0078425) Train Loss: 6.2960, Train Steps/Sec: 0.91 +[2025-04-24 23:13:47] (step=0078450) Train Loss: 6.2917, Train Steps/Sec: 0.90 +[2025-04-24 23:14:14] (step=0078475) Train Loss: 6.3275, Train Steps/Sec: 0.90 +[2025-04-24 23:14:42] (step=0078500) Train Loss: 6.3280, Train Steps/Sec: 0.90 +[2025-04-24 23:15:10] (step=0078525) Train Loss: 6.3558, Train Steps/Sec: 0.90 +[2025-04-24 23:15:37] (step=0078550) Train Loss: 6.2865, Train Steps/Sec: 0.90 +[2025-04-24 23:16:05] (step=0078575) Train Loss: 6.2856, Train Steps/Sec: 0.90 +[2025-04-24 23:16:33] (step=0078600) Train Loss: 6.3417, Train Steps/Sec: 0.90 +[2025-04-24 23:17:01] (step=0078625) Train Loss: 6.3110, Train Steps/Sec: 0.90 +[2025-04-24 23:17:28] (step=0078650) Train Loss: 6.2943, Train Steps/Sec: 0.90 +[2025-04-24 23:17:56] (step=0078675) Train Loss: 6.3399, Train Steps/Sec: 0.91 +[2025-04-24 23:18:24] (step=0078700) Train Loss: 6.3091, Train Steps/Sec: 0.89 +[2025-04-24 23:18:52] (step=0078725) Train Loss: 6.3275, Train Steps/Sec: 0.90 +[2025-04-24 23:19:19] (step=0078750) Train Loss: 6.3043, Train Steps/Sec: 0.90 +[2025-04-24 23:19:47] (step=0078775) Train Loss: 6.3055, Train Steps/Sec: 0.90 +[2025-04-24 23:20:15] (step=0078800) Train Loss: 6.2898, Train Steps/Sec: 0.90 +[2025-04-24 23:20:43] (step=0078825) Train Loss: 6.3370, Train Steps/Sec: 0.90 +[2025-04-24 23:21:10] (step=0078850) Train Loss: 6.3222, Train Steps/Sec: 0.90 +[2025-04-24 23:21:38] (step=0078875) Train Loss: 6.2976, Train Steps/Sec: 0.90 +[2025-04-24 23:22:06] (step=0078900) Train Loss: 6.3148, Train Steps/Sec: 0.90 +[2025-04-24 23:22:34] (step=0078925) Train Loss: 6.3127, Train Steps/Sec: 0.90 +[2025-04-24 23:23:01] (step=0078950) Train Loss: 6.3443, Train Steps/Sec: 0.90 +[2025-04-24 23:23:29] (step=0078975) Train Loss: 6.2860, Train Steps/Sec: 0.91 +[2025-04-24 23:23:57] (step=0079000) Train Loss: 6.2488, Train Steps/Sec: 0.90 +[2025-04-24 23:24:24] (step=0079025) Train Loss: 6.2944, Train Steps/Sec: 0.91 +[2025-04-24 23:24:52] (step=0079050) Train Loss: 6.2851, Train Steps/Sec: 0.91 +[2025-04-24 23:25:20] (step=0079075) Train Loss: 6.3149, Train Steps/Sec: 0.90 +[2025-04-24 23:25:47] (step=0079100) Train Loss: 6.3478, Train Steps/Sec: 0.90 +[2025-04-24 23:26:15] (step=0079125) Train Loss: 6.2965, Train Steps/Sec: 0.91 +[2025-04-24 23:26:43] (step=0079150) Train Loss: 6.3112, Train Steps/Sec: 0.91 +[2025-04-24 23:27:10] (step=0079175) Train Loss: 6.3248, Train Steps/Sec: 0.90 +[2025-04-24 23:27:38] (step=0079200) Train Loss: 6.2991, Train Steps/Sec: 0.90 +[2025-04-24 23:28:06] (step=0079225) Train Loss: 6.3004, Train Steps/Sec: 0.90 +[2025-04-24 23:28:33] (step=0079250) Train Loss: 6.2923, Train Steps/Sec: 0.90 +[2025-04-24 23:29:01] (step=0079275) Train Loss: 6.3150, Train Steps/Sec: 0.90 +[2025-04-24 23:29:29] (step=0079300) Train Loss: 6.2325, Train Steps/Sec: 0.90 +[2025-04-24 23:29:57] (step=0079325) Train Loss: 6.2862, Train Steps/Sec: 0.90 +[2025-04-24 23:30:24] (step=0079350) Train Loss: 6.2900, Train Steps/Sec: 0.90 +[2025-04-24 23:30:52] (step=0079375) Train Loss: 6.3329, Train Steps/Sec: 0.90 +[2025-04-24 23:31:20] (step=0079400) Train Loss: 6.2989, Train Steps/Sec: 0.90 +[2025-04-24 23:31:53] (step=0079425) Train Loss: 6.3405, Train Steps/Sec: 0.77 +[2025-04-24 23:32:20] (step=0079450) Train Loss: 6.3465, Train Steps/Sec: 0.90 +[2025-04-24 23:32:48] (step=0079475) Train Loss: 6.3175, Train Steps/Sec: 0.90 +[2025-04-24 23:33:16] (step=0079500) Train Loss: 6.2847, Train Steps/Sec: 0.90 +[2025-04-24 23:33:43] (step=0079525) Train Loss: 6.2775, Train Steps/Sec: 0.90 +[2025-04-24 23:34:11] (step=0079550) Train Loss: 6.2837, Train Steps/Sec: 0.90 +[2025-04-24 23:34:39] (step=0079575) Train Loss: 6.3318, Train Steps/Sec: 0.90 +[2025-04-24 23:35:07] (step=0079600) Train Loss: 6.2265, Train Steps/Sec: 0.90 +[2025-04-24 23:35:39] (step=0079625) Train Loss: 6.3514, Train Steps/Sec: 0.77 +[2025-04-24 23:36:07] (step=0079650) Train Loss: 6.2594, Train Steps/Sec: 0.90 +[2025-04-24 23:36:35] (step=0079675) Train Loss: 6.2761, Train Steps/Sec: 0.90 +[2025-04-24 23:37:02] (step=0079700) Train Loss: 6.3271, Train Steps/Sec: 0.90 +[2025-04-24 23:37:35] (step=0079725) Train Loss: 6.3209, Train Steps/Sec: 0.76 +[2025-04-24 23:38:03] (step=0079750) Train Loss: 6.2610, Train Steps/Sec: 0.90 +[2025-04-24 23:38:31] (step=0079775) Train Loss: 6.2846, Train Steps/Sec: 0.90 +[2025-04-24 23:38:58] (step=0079800) Train Loss: 6.2309, Train Steps/Sec: 0.90 +[2025-04-24 23:39:26] (step=0079825) Train Loss: 6.2704, Train Steps/Sec: 0.91 +[2025-04-24 23:39:59] (step=0079850) Train Loss: 6.2802, Train Steps/Sec: 0.76 +[2025-04-24 23:40:32] (step=0079875) Train Loss: 6.2863, Train Steps/Sec: 0.76 +[2025-04-24 23:41:05] (step=0079900) Train Loss: 6.3426, Train Steps/Sec: 0.76 +[2025-04-24 23:41:32] (step=0079925) Train Loss: 6.2735, Train Steps/Sec: 0.90 +[2025-04-24 23:42:00] (step=0079950) Train Loss: 6.2970, Train Steps/Sec: 0.90 +[2025-04-24 23:42:28] (step=0079975) Train Loss: 6.2832, Train Steps/Sec: 0.90 +[2025-04-24 23:42:55] (step=0080000) Train Loss: 6.3344, Train Steps/Sec: 0.90 +[2025-04-24 23:42:56] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 23:42:56] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 23:42:56] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 23:47:40] Finish Eval in 80000 steps... +[2025-04-24 23:48:01] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0080000.pt +[2025-04-24 23:48:03] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0078000.pt +[2025-04-24 23:48:31] (step=0080025) Train Loss: 6.2396, Train Steps/Sec: 0.07 +[2025-04-24 23:48:59] (step=0080050) Train Loss: 6.2821, Train Steps/Sec: 0.90 +[2025-04-24 23:49:27] (step=0080075) Train Loss: 6.3444, Train Steps/Sec: 0.90 +[2025-04-24 23:49:55] (step=0080100) Train Loss: 6.3113, Train Steps/Sec: 0.89 +[2025-04-24 23:50:22] (step=0080125) Train Loss: 6.2987, Train Steps/Sec: 0.90 +[2025-04-24 23:50:50] (step=0080150) Train Loss: 6.3200, Train Steps/Sec: 0.90 +[2025-04-24 23:51:18] (step=0080175) Train Loss: 6.3247, Train Steps/Sec: 0.90 +[2025-04-24 23:51:46] (step=0080200) Train Loss: 6.2890, Train Steps/Sec: 0.90 +[2025-04-24 23:52:13] (step=0080225) Train Loss: 6.2853, Train Steps/Sec: 0.90 +[2025-04-24 23:52:41] (step=0080250) Train Loss: 6.2797, Train Steps/Sec: 0.91 +[2025-04-24 23:53:09] (step=0080275) Train Loss: 6.3013, Train Steps/Sec: 0.90 +[2025-04-24 23:53:36] (step=0080300) Train Loss: 6.2952, Train Steps/Sec: 0.90 +[2025-04-24 23:54:04] (step=0080325) Train Loss: 6.3080, Train Steps/Sec: 0.90 +[2025-04-24 23:54:32] (step=0080350) Train Loss: 6.3039, Train Steps/Sec: 0.90 +[2025-04-24 23:55:00] (step=0080375) Train Loss: 6.3413, Train Steps/Sec: 0.90 +[2025-04-24 23:55:27] (step=0080400) Train Loss: 6.3499, Train Steps/Sec: 0.90 +[2025-04-24 23:55:55] (step=0080425) Train Loss: 6.3143, Train Steps/Sec: 0.91 +[2025-04-24 23:56:23] (step=0080450) Train Loss: 6.3505, Train Steps/Sec: 0.90 +[2025-04-24 23:56:51] (step=0080475) Train Loss: 6.3251, Train Steps/Sec: 0.90 +[2025-04-24 23:57:18] (step=0080500) Train Loss: 6.2631, Train Steps/Sec: 0.90 +[2025-04-24 23:57:46] (step=0080525) Train Loss: 6.2819, Train Steps/Sec: 0.90 +[2025-04-24 23:58:14] (step=0080550) Train Loss: 6.2579, Train Steps/Sec: 0.90 +[2025-04-24 23:58:41] (step=0080575) Train Loss: 6.3120, Train Steps/Sec: 0.90 +[2025-04-24 23:59:09] (step=0080600) Train Loss: 6.2842, Train Steps/Sec: 0.90 +[2025-04-24 23:59:37] (step=0080625) Train Loss: 6.2931, Train Steps/Sec: 0.90 +[2025-04-25 00:00:05] (step=0080650) Train Loss: 6.2838, Train Steps/Sec: 0.90 +[2025-04-25 00:00:32] (step=0080675) Train Loss: 6.2908, Train Steps/Sec: 0.90 +[2025-04-25 00:01:00] (step=0080700) Train Loss: 6.2675, Train Steps/Sec: 0.89 +[2025-04-25 00:01:28] (step=0080725) Train Loss: 6.3007, Train Steps/Sec: 0.90 +[2025-04-25 00:01:56] (step=0080750) Train Loss: 6.3461, Train Steps/Sec: 0.91 +[2025-04-25 00:02:23] (step=0080775) Train Loss: 6.2940, Train Steps/Sec: 0.90 +[2025-04-25 00:02:51] (step=0080800) Train Loss: 6.3360, Train Steps/Sec: 0.90 +[2025-04-25 00:03:19] (step=0080825) Train Loss: 6.3183, Train Steps/Sec: 0.91 +[2025-04-25 00:03:46] (step=0080850) Train Loss: 6.2951, Train Steps/Sec: 0.90 +[2025-04-25 00:04:14] (step=0080875) Train Loss: 6.2900, Train Steps/Sec: 0.90 +[2025-04-25 00:04:42] (step=0080900) Train Loss: 6.3014, Train Steps/Sec: 0.90 +[2025-04-25 00:05:09] (step=0080925) Train Loss: 6.3155, Train Steps/Sec: 0.91 +[2025-04-25 00:05:37] (step=0080950) Train Loss: 6.2947, Train Steps/Sec: 0.90 +[2025-04-25 00:06:05] (step=0080975) Train Loss: 6.2528, Train Steps/Sec: 0.90 +[2025-04-25 00:06:33] (step=0081000) Train Loss: 6.2862, Train Steps/Sec: 0.90 +[2025-04-25 00:07:00] (step=0081025) Train Loss: 6.2918, Train Steps/Sec: 0.91 +[2025-04-25 00:07:28] (step=0081050) Train Loss: 6.3145, Train Steps/Sec: 0.90 +[2025-04-25 00:07:56] (step=0081075) Train Loss: 6.2846, Train Steps/Sec: 0.90 +[2025-04-25 00:08:24] (step=0081100) Train Loss: 6.3027, Train Steps/Sec: 0.90 +[2025-04-25 00:08:51] (step=0081125) Train Loss: 6.3519, Train Steps/Sec: 0.90 +[2025-04-25 00:09:19] (step=0081150) Train Loss: 6.3011, Train Steps/Sec: 0.90 +[2025-04-25 00:09:47] (step=0081175) Train Loss: 6.3314, Train Steps/Sec: 0.90 +[2025-04-25 00:10:15] (step=0081200) Train Loss: 6.2779, Train Steps/Sec: 0.90 +[2025-04-25 00:10:42] (step=0081225) Train Loss: 6.3105, Train Steps/Sec: 0.90 +[2025-04-25 00:11:10] (step=0081250) Train Loss: 6.2812, Train Steps/Sec: 0.90 +[2025-04-25 00:11:38] (step=0081275) Train Loss: 6.3224, Train Steps/Sec: 0.90 +[2025-04-25 00:12:06] (step=0081300) Train Loss: 6.3541, Train Steps/Sec: 0.90 +[2025-04-25 00:12:33] (step=0081325) Train Loss: 6.2901, Train Steps/Sec: 0.91 +[2025-04-25 00:13:01] (step=0081350) Train Loss: 6.3214, Train Steps/Sec: 0.91 +[2025-04-25 00:13:28] (step=0081375) Train Loss: 6.2894, Train Steps/Sec: 0.91 +[2025-04-25 00:13:56] (step=0081400) Train Loss: 6.2600, Train Steps/Sec: 0.90 +[2025-04-25 00:14:24] (step=0081425) Train Loss: 6.2929, Train Steps/Sec: 0.91 +[2025-04-25 00:14:51] (step=0081450) Train Loss: 6.3048, Train Steps/Sec: 0.91 +[2025-04-25 00:15:19] (step=0081475) Train Loss: 6.3243, Train Steps/Sec: 0.90 +[2025-04-25 00:15:47] (step=0081500) Train Loss: 6.3267, Train Steps/Sec: 0.90 +[2025-04-25 00:16:14] (step=0081525) Train Loss: 6.3252, Train Steps/Sec: 0.91 +[2025-04-25 00:16:42] (step=0081550) Train Loss: 6.3091, Train Steps/Sec: 0.90 +[2025-04-25 00:17:10] (step=0081575) Train Loss: 6.2037, Train Steps/Sec: 0.91 +[2025-04-25 00:17:43] (step=0081600) Train Loss: 6.3330, Train Steps/Sec: 0.75 +[2025-04-25 00:18:11] (step=0081625) Train Loss: 6.2705, Train Steps/Sec: 0.90 +[2025-04-25 00:18:38] (step=0081650) Train Loss: 6.3199, Train Steps/Sec: 0.91 +[2025-04-25 00:19:06] (step=0081675) Train Loss: 6.2925, Train Steps/Sec: 0.91 +[2025-04-25 00:19:34] (step=0081700) Train Loss: 6.3071, Train Steps/Sec: 0.90 +[2025-04-25 00:20:01] (step=0081725) Train Loss: 6.2793, Train Steps/Sec: 0.90 +[2025-04-25 00:20:29] (step=0081750) Train Loss: 6.3074, Train Steps/Sec: 0.91 +[2025-04-25 00:20:56] (step=0081775) Train Loss: 6.2620, Train Steps/Sec: 0.90 +[2025-04-25 00:21:24] (step=0081800) Train Loss: 6.3373, Train Steps/Sec: 0.90 +[2025-04-25 00:21:52] (step=0081825) Train Loss: 6.3354, Train Steps/Sec: 0.90 +[2025-04-25 00:22:20] (step=0081850) Train Loss: 6.3243, Train Steps/Sec: 0.90 +[2025-04-25 00:22:47] (step=0081875) Train Loss: 6.2798, Train Steps/Sec: 0.90 +[2025-04-25 00:23:15] (step=0081900) Train Loss: 6.2984, Train Steps/Sec: 0.90 +[2025-04-25 00:23:43] (step=0081925) Train Loss: 6.2479, Train Steps/Sec: 0.90 +[2025-04-25 00:24:10] (step=0081950) Train Loss: 6.3009, Train Steps/Sec: 0.90 +[2025-04-25 00:24:38] (step=0081975) Train Loss: 6.2692, Train Steps/Sec: 0.90 +[2025-04-25 00:25:06] (step=0082000) Train Loss: 6.2803, Train Steps/Sec: 0.90 +[2025-04-25 00:25:06] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-25 00:25:06] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-25 00:25:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 00:29:51] Finish Eval in 82000 steps... +[2025-04-25 00:30:13] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0082000.pt +[2025-04-25 00:30:15] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0080000.pt +[2025-04-25 00:30:43] (step=0082025) Train Loss: 6.2842, Train Steps/Sec: 0.07 +[2025-04-25 00:31:10] (step=0082050) Train Loss: 6.2654, Train Steps/Sec: 0.90 +[2025-04-25 00:31:38] (step=0082075) Train Loss: 6.2760, Train Steps/Sec: 0.91 +[2025-04-25 00:32:06] (step=0082100) Train Loss: 6.2827, Train Steps/Sec: 0.90 +[2025-04-25 00:32:34] (step=0082125) Train Loss: 6.3121, Train Steps/Sec: 0.90 +[2025-04-25 00:33:01] (step=0082150) Train Loss: 6.2911, Train Steps/Sec: 0.90 +[2025-04-25 00:33:29] (step=0082175) Train Loss: 6.2983, Train Steps/Sec: 0.91 +[2025-04-25 00:33:57] (step=0082200) Train Loss: 6.2521, Train Steps/Sec: 0.90 +[2025-04-25 00:34:24] (step=0082225) Train Loss: 6.3287, Train Steps/Sec: 0.90 +[2025-04-25 00:34:52] (step=0082250) Train Loss: 6.3299, Train Steps/Sec: 0.90 +[2025-04-25 00:35:20] (step=0082275) Train Loss: 6.2717, Train Steps/Sec: 0.91 +[2025-04-25 00:35:47] (step=0082300) Train Loss: 6.3402, Train Steps/Sec: 0.90 +[2025-04-25 00:36:15] (step=0082325) Train Loss: 6.2813, Train Steps/Sec: 0.90 +[2025-04-25 00:36:42] (step=0082350) Train Loss: 6.3512, Train Steps/Sec: 0.91 +[2025-04-25 00:37:10] (step=0082375) Train Loss: 6.3024, Train Steps/Sec: 0.90 +[2025-04-25 00:37:38] (step=0082400) Train Loss: 6.2664, Train Steps/Sec: 0.90 +[2025-04-25 00:38:05] (step=0082425) Train Loss: 6.2551, Train Steps/Sec: 0.91 +[2025-04-25 00:38:33] (step=0082450) Train Loss: 6.3076, Train Steps/Sec: 0.90 +[2025-04-25 00:39:01] (step=0082475) Train Loss: 6.2789, Train Steps/Sec: 0.90 +[2025-04-25 00:39:29] (step=0082500) Train Loss: 6.3366, Train Steps/Sec: 0.90 +[2025-04-25 00:39:56] (step=0082525) Train Loss: 6.2961, Train Steps/Sec: 0.91 +[2025-04-25 00:40:24] (step=0082550) Train Loss: 6.2779, Train Steps/Sec: 0.90 +[2025-04-25 00:40:52] (step=0082575) Train Loss: 6.2716, Train Steps/Sec: 0.90 +[2025-04-25 00:41:20] (step=0082600) Train Loss: 6.3244, Train Steps/Sec: 0.89 +[2025-04-25 00:41:47] (step=0082625) Train Loss: 6.3539, Train Steps/Sec: 0.90 +[2025-04-25 00:42:15] (step=0082650) Train Loss: 6.3210, Train Steps/Sec: 0.90 +[2025-04-25 00:42:43] (step=0082675) Train Loss: 6.2843, Train Steps/Sec: 0.90 +[2025-04-25 00:43:11] (step=0082700) Train Loss: 6.2763, Train Steps/Sec: 0.90 +[2025-04-25 00:43:38] (step=0082725) Train Loss: 6.2562, Train Steps/Sec: 0.91 +[2025-04-25 00:44:06] (step=0082750) Train Loss: 6.3079, Train Steps/Sec: 0.90 +[2025-04-25 00:44:34] (step=0082775) Train Loss: 6.3034, Train Steps/Sec: 0.90 +[2025-04-25 00:45:01] (step=0082800) Train Loss: 6.2875, Train Steps/Sec: 0.90 +[2025-04-25 00:45:29] (step=0082825) Train Loss: 6.3148, Train Steps/Sec: 0.90 +[2025-04-25 00:45:57] (step=0082850) Train Loss: 6.2617, Train Steps/Sec: 0.91 +[2025-04-25 00:46:24] (step=0082875) Train Loss: 6.2635, Train Steps/Sec: 0.90 +[2025-04-25 00:46:52] (step=0082900) Train Loss: 6.3300, Train Steps/Sec: 0.90 +[2025-04-25 00:47:20] (step=0082925) Train Loss: 6.2996, Train Steps/Sec: 0.90 +[2025-04-25 00:47:48] (step=0082950) Train Loss: 6.3010, Train Steps/Sec: 0.90 +[2025-04-25 00:48:15] (step=0082975) Train Loss: 6.2986, Train Steps/Sec: 0.90 +[2025-04-25 00:48:43] (step=0083000) Train Loss: 6.2627, Train Steps/Sec: 0.90 +[2025-04-25 00:49:11] (step=0083025) Train Loss: 6.2760, Train Steps/Sec: 0.91 +[2025-04-25 00:49:39] (step=0083050) Train Loss: 6.2924, Train Steps/Sec: 0.90 +[2025-04-25 00:50:06] (step=0083075) Train Loss: 6.2975, Train Steps/Sec: 0.90 +[2025-04-25 00:50:34] (step=0083100) Train Loss: 6.3488, Train Steps/Sec: 0.90 +[2025-04-25 00:51:02] (step=0083125) Train Loss: 6.3255, Train Steps/Sec: 0.90 +[2025-04-25 00:51:29] (step=0083150) Train Loss: 6.3039, Train Steps/Sec: 0.90 +[2025-04-25 00:51:57] (step=0083175) Train Loss: 6.2618, Train Steps/Sec: 0.90 +[2025-04-25 00:52:25] (step=0083200) Train Loss: 6.3001, Train Steps/Sec: 0.89 +[2025-04-25 00:52:53] (step=0083225) Train Loss: 6.3143, Train Steps/Sec: 0.90 +[2025-04-25 00:53:21] (step=0083250) Train Loss: 6.2765, Train Steps/Sec: 0.90 +[2025-04-25 00:53:48] (step=0083275) Train Loss: 6.2852, Train Steps/Sec: 0.90 +[2025-04-25 00:54:16] (step=0083300) Train Loss: 6.2621, Train Steps/Sec: 0.90 +[2025-04-25 00:54:44] (step=0083325) Train Loss: 6.2605, Train Steps/Sec: 0.90 +[2025-04-25 00:55:12] (step=0083350) Train Loss: 6.2825, Train Steps/Sec: 0.90 +[2025-04-25 00:55:39] (step=0083375) Train Loss: 6.3069, Train Steps/Sec: 0.90 +[2025-04-25 00:56:07] (step=0083400) Train Loss: 6.2700, Train Steps/Sec: 0.90 +[2025-04-25 00:56:35] (step=0083425) Train Loss: 6.3034, Train Steps/Sec: 0.91 +[2025-04-25 00:57:02] (step=0083450) Train Loss: 6.2897, Train Steps/Sec: 0.90 +[2025-04-25 00:57:30] (step=0083475) Train Loss: 6.2665, Train Steps/Sec: 0.90 +[2025-04-25 00:57:58] (step=0083500) Train Loss: 6.2871, Train Steps/Sec: 0.90 +[2025-04-25 00:58:26] (step=0083525) Train Loss: 6.2525, Train Steps/Sec: 0.90 +[2025-04-25 00:58:53] (step=0083550) Train Loss: 6.2795, Train Steps/Sec: 0.91 +[2025-04-25 00:59:21] (step=0083575) Train Loss: 6.3481, Train Steps/Sec: 0.90 +[2025-04-25 00:59:49] (step=0083600) Train Loss: 6.2949, Train Steps/Sec: 0.90 +[2025-04-25 01:00:17] (step=0083625) Train Loss: 6.2795, Train Steps/Sec: 0.90 +[2025-04-25 01:00:44] (step=0083650) Train Loss: 6.2985, Train Steps/Sec: 0.90 +[2025-04-25 01:01:12] (step=0083675) Train Loss: 6.2637, Train Steps/Sec: 0.90 +[2025-04-25 01:01:40] (step=0083700) Train Loss: 6.2954, Train Steps/Sec: 0.90 +[2025-04-25 01:02:07] (step=0083725) Train Loss: 6.2958, Train Steps/Sec: 0.91 +[2025-04-25 01:02:35] (step=0083750) Train Loss: 6.3142, Train Steps/Sec: 0.91 +[2025-04-25 01:03:03] (step=0083775) Train Loss: 6.3060, Train Steps/Sec: 0.90 +[2025-04-25 01:03:31] (step=0083800) Train Loss: 6.2823, Train Steps/Sec: 0.90 +[2025-04-25 01:03:58] (step=0083825) Train Loss: 6.2844, Train Steps/Sec: 0.90 +[2025-04-25 01:04:26] (step=0083850) Train Loss: 6.2959, Train Steps/Sec: 0.90 +[2025-04-25 01:04:54] (step=0083875) Train Loss: 6.2746, Train Steps/Sec: 0.90 +[2025-04-25 01:05:21] (step=0083900) Train Loss: 6.2624, Train Steps/Sec: 0.90 +[2025-04-25 01:05:49] (step=0083925) Train Loss: 6.3060, Train Steps/Sec: 0.90 +[2025-04-25 01:06:17] (step=0083950) Train Loss: 6.3230, Train Steps/Sec: 0.90 +[2025-04-25 01:06:45] (step=0083975) Train Loss: 6.3253, Train Steps/Sec: 0.90 +[2025-04-25 01:07:13] (step=0084000) Train Loss: 6.3105, Train Steps/Sec: 0.90 +[2025-04-25 01:07:13] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-25 01:07:13] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-25 01:07:13] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 01:11:57] Finish Eval in 84000 steps... +[2025-04-25 01:12:18] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0084000.pt +[2025-04-25 01:12:20] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0082000.pt +[2025-04-25 01:12:48] (step=0084025) Train Loss: 6.3277, Train Steps/Sec: 0.07 +[2025-04-25 01:13:16] (step=0084050) Train Loss: 6.3036, Train Steps/Sec: 0.90 +[2025-04-25 01:13:43] (step=0084075) Train Loss: 6.2914, Train Steps/Sec: 0.90 +[2025-04-25 01:14:11] (step=0084100) Train Loss: 6.2925, Train Steps/Sec: 0.90 +[2025-04-25 01:14:39] (step=0084125) Train Loss: 6.3324, Train Steps/Sec: 0.90 +[2025-04-25 01:15:06] (step=0084150) Train Loss: 6.2947, Train Steps/Sec: 0.90 +[2025-04-25 01:15:34] (step=0084175) Train Loss: 6.2639, Train Steps/Sec: 0.90 +[2025-04-25 01:16:02] (step=0084200) Train Loss: 6.2901, Train Steps/Sec: 0.89 +[2025-04-25 01:16:30] (step=0084225) Train Loss: 6.3247, Train Steps/Sec: 0.91 +[2025-04-25 01:17:02] (step=0084250) Train Loss: 6.2496, Train Steps/Sec: 0.77 +[2025-04-25 01:17:30] (step=0084275) Train Loss: 6.3018, Train Steps/Sec: 0.90 +[2025-04-25 01:18:03] (step=0084300) Train Loss: 6.2970, Train Steps/Sec: 0.76 +[2025-04-25 01:18:30] (step=0084325) Train Loss: 6.3169, Train Steps/Sec: 0.91 +[2025-04-25 01:18:58] (step=0084350) Train Loss: 6.3159, Train Steps/Sec: 0.90 +[2025-04-25 01:19:26] (step=0084375) Train Loss: 6.2599, Train Steps/Sec: 0.90 +[2025-04-25 01:19:59] (step=0084400) Train Loss: 6.2958, Train Steps/Sec: 0.76 +[2025-04-25 01:20:26] (step=0084425) Train Loss: 6.3050, Train Steps/Sec: 0.90 +[2025-04-25 01:20:54] (step=0084450) Train Loss: 6.2989, Train Steps/Sec: 0.91 +[2025-04-25 01:21:21] (step=0084475) Train Loss: 6.3016, Train Steps/Sec: 0.91 +[2025-04-25 01:21:49] (step=0084500) Train Loss: 6.2918, Train Steps/Sec: 0.90 +[2025-04-25 01:22:17] (step=0084525) Train Loss: 6.2981, Train Steps/Sec: 0.90 +[2025-04-25 01:22:45] (step=0084550) Train Loss: 6.3001, Train Steps/Sec: 0.90 +[2025-04-25 01:23:12] (step=0084575) Train Loss: 6.3148, Train Steps/Sec: 0.91 +[2025-04-25 01:23:45] (step=0084600) Train Loss: 6.3045, Train Steps/Sec: 0.76 +[2025-04-25 01:24:18] (step=0084625) Train Loss: 6.2882, Train Steps/Sec: 0.76 +[2025-04-25 01:24:45] (step=0084650) Train Loss: 6.2856, Train Steps/Sec: 0.91 +[2025-04-25 01:25:18] (step=0084675) Train Loss: 6.2846, Train Steps/Sec: 0.76 +[2025-04-25 01:25:47] (step=0084700) Train Loss: 6.3095, Train Steps/Sec: 0.88 +[2025-04-25 01:26:14] (step=0084725) Train Loss: 6.2786, Train Steps/Sec: 0.90 +[2025-04-25 01:26:42] (step=0084750) Train Loss: 6.2286, Train Steps/Sec: 0.90 +[2025-04-25 01:27:10] (step=0084775) Train Loss: 6.2970, Train Steps/Sec: 0.90 +[2025-04-25 01:27:38] (step=0084800) Train Loss: 6.3282, Train Steps/Sec: 0.90 +[2025-04-25 01:28:05] (step=0084825) Train Loss: 6.3397, Train Steps/Sec: 0.90 +[2025-04-25 01:28:33] (step=0084850) Train Loss: 6.2799, Train Steps/Sec: 0.90 +[2025-04-25 01:29:01] (step=0084875) Train Loss: 6.3790, Train Steps/Sec: 0.90 +[2025-04-25 01:29:28] (step=0084900) Train Loss: 6.3057, Train Steps/Sec: 0.90 +[2025-04-25 01:29:56] (step=0084925) Train Loss: 6.2720, Train Steps/Sec: 0.90 +[2025-04-25 01:30:24] (step=0084950) Train Loss: 6.2597, Train Steps/Sec: 0.90 +[2025-04-25 01:30:51] (step=0084975) Train Loss: 6.2932, Train Steps/Sec: 0.90 +[2025-04-25 01:31:19] (step=0085000) Train Loss: 6.3215, Train Steps/Sec: 0.90 +[2025-04-25 01:31:47] (step=0085025) Train Loss: 6.2803, Train Steps/Sec: 0.90 +[2025-04-25 01:32:15] (step=0085050) Train Loss: 6.3027, Train Steps/Sec: 0.90 +[2025-04-25 01:32:42] (step=0085075) Train Loss: 6.2354, Train Steps/Sec: 0.91 +[2025-04-25 01:33:10] (step=0085100) Train Loss: 6.3276, Train Steps/Sec: 0.90 +[2025-04-25 01:33:38] (step=0085125) Train Loss: 6.3194, Train Steps/Sec: 0.90 +[2025-04-25 01:34:05] (step=0085150) Train Loss: 6.3059, Train Steps/Sec: 0.90 +[2025-04-25 01:34:33] (step=0085175) Train Loss: 6.3280, Train Steps/Sec: 0.90 +[2025-04-25 01:35:01] (step=0085200) Train Loss: 6.3355, Train Steps/Sec: 0.90 +[2025-04-25 01:35:29] (step=0085225) Train Loss: 6.3105, Train Steps/Sec: 0.90 +[2025-04-25 01:35:56] (step=0085250) Train Loss: 6.2949, Train Steps/Sec: 0.91 +[2025-04-25 01:36:24] (step=0085275) Train Loss: 6.3008, Train Steps/Sec: 0.90 +[2025-04-25 01:36:52] (step=0085300) Train Loss: 6.3230, Train Steps/Sec: 0.90 +[2025-04-25 01:37:19] (step=0085325) Train Loss: 6.3097, Train Steps/Sec: 0.90 +[2025-04-25 01:37:47] (step=0085350) Train Loss: 6.3685, Train Steps/Sec: 0.90 +[2025-04-25 01:38:15] (step=0085375) Train Loss: 6.3726, Train Steps/Sec: 0.90 +[2025-04-25 01:38:43] (step=0085400) Train Loss: 6.2958, Train Steps/Sec: 0.90 +[2025-04-25 01:39:10] (step=0085425) Train Loss: 6.2952, Train Steps/Sec: 0.90 +[2025-04-25 01:39:38] (step=0085450) Train Loss: 6.2994, Train Steps/Sec: 0.90 +[2025-04-25 01:40:06] (step=0085475) Train Loss: 6.3279, Train Steps/Sec: 0.90 +[2025-04-25 01:40:34] (step=0085500) Train Loss: 6.2988, Train Steps/Sec: 0.89 +[2025-04-25 01:41:01] (step=0085525) Train Loss: 6.3568, Train Steps/Sec: 0.90 +[2025-04-25 01:41:29] (step=0085550) Train Loss: 6.2656, Train Steps/Sec: 0.90 +[2025-04-25 01:41:57] (step=0085575) Train Loss: 6.2918, Train Steps/Sec: 0.91 +[2025-04-25 01:42:24] (step=0085600) Train Loss: 6.3262, Train Steps/Sec: 0.90 +[2025-04-25 01:42:52] (step=0085625) Train Loss: 6.2828, Train Steps/Sec: 0.91 +[2025-04-25 01:43:20] (step=0085650) Train Loss: 6.2726, Train Steps/Sec: 0.90 +[2025-04-25 01:43:47] (step=0085675) Train Loss: 6.2412, Train Steps/Sec: 0.91 +[2025-04-25 01:44:15] (step=0085700) Train Loss: 6.2661, Train Steps/Sec: 0.90 +[2025-04-25 01:44:43] (step=0085725) Train Loss: 6.2798, Train Steps/Sec: 0.90 +[2025-04-25 01:45:10] (step=0085750) Train Loss: 6.2791, Train Steps/Sec: 0.90 +[2025-04-25 01:45:38] (step=0085775) Train Loss: 6.2548, Train Steps/Sec: 0.91 +[2025-04-25 01:46:06] (step=0085800) Train Loss: 6.3184, Train Steps/Sec: 0.90 +[2025-04-25 01:46:34] (step=0085825) Train Loss: 6.3161, Train Steps/Sec: 0.90 +[2025-04-25 01:47:01] (step=0085850) Train Loss: 6.3301, Train Steps/Sec: 0.91 +[2025-04-25 01:47:29] (step=0085875) Train Loss: 6.2779, Train Steps/Sec: 0.90 +[2025-04-25 01:47:57] (step=0085900) Train Loss: 6.3103, Train Steps/Sec: 0.90 +[2025-04-25 01:48:24] (step=0085925) Train Loss: 6.3249, Train Steps/Sec: 0.90 +[2025-04-25 01:48:52] (step=0085950) Train Loss: 6.3075, Train Steps/Sec: 0.90 +[2025-04-25 01:49:20] (step=0085975) Train Loss: 6.3152, Train Steps/Sec: 0.90 +[2025-04-25 01:49:48] (step=0086000) Train Loss: 6.2855, Train Steps/Sec: 0.90 +[2025-04-25 01:49:48] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-25 01:49:48] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-25 01:49:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 01:54:33] Finish Eval in 86000 steps... +[2025-04-25 01:54:55] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0086000.pt +[2025-04-25 01:54:57] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0084000.pt +[2025-04-25 01:55:25] (step=0086025) Train Loss: 6.2775, Train Steps/Sec: 0.07 +[2025-04-25 01:55:53] (step=0086050) Train Loss: 6.2732, Train Steps/Sec: 0.90 +[2025-04-25 01:56:20] (step=0086075) Train Loss: 6.3008, Train Steps/Sec: 0.91 +[2025-04-25 01:56:48] (step=0086100) Train Loss: 6.3039, Train Steps/Sec: 0.90 +[2025-04-25 01:57:16] (step=0086125) Train Loss: 6.3156, Train Steps/Sec: 0.91 +[2025-04-25 01:57:43] (step=0086150) Train Loss: 6.3149, Train Steps/Sec: 0.90 +[2025-04-25 01:58:11] (step=0086175) Train Loss: 6.3013, Train Steps/Sec: 0.90 +[2025-04-25 01:58:39] (step=0086200) Train Loss: 6.2565, Train Steps/Sec: 0.90 +[2025-04-25 01:59:06] (step=0086225) Train Loss: 6.2862, Train Steps/Sec: 0.91 +[2025-04-25 01:59:34] (step=0086250) Train Loss: 6.2950, Train Steps/Sec: 0.91 +[2025-04-25 02:00:02] (step=0086275) Train Loss: 6.3030, Train Steps/Sec: 0.90 +[2025-04-25 02:00:29] (step=0086300) Train Loss: 6.2941, Train Steps/Sec: 0.90 +[2025-04-25 02:00:57] (step=0086325) Train Loss: 6.3046, Train Steps/Sec: 0.90 +[2025-04-25 02:01:25] (step=0086350) Train Loss: 6.2654, Train Steps/Sec: 0.90 +[2025-04-25 02:01:53] (step=0086375) Train Loss: 6.3108, Train Steps/Sec: 0.90 +[2025-04-25 02:02:21] (step=0086400) Train Loss: 6.3832, Train Steps/Sec: 0.90 +[2025-04-25 02:02:48] (step=0086425) Train Loss: 6.3336, Train Steps/Sec: 0.90 +[2025-04-25 02:03:16] (step=0086450) Train Loss: 6.2664, Train Steps/Sec: 0.91 +[2025-04-25 02:03:44] (step=0086475) Train Loss: 6.3266, Train Steps/Sec: 0.90 +[2025-04-25 02:04:11] (step=0086500) Train Loss: 6.2752, Train Steps/Sec: 0.90 +[2025-04-25 02:04:39] (step=0086525) Train Loss: 6.3390, Train Steps/Sec: 0.91 +[2025-04-25 02:05:07] (step=0086550) Train Loss: 6.3030, Train Steps/Sec: 0.90 +[2025-04-25 02:05:34] (step=0086575) Train Loss: 6.2414, Train Steps/Sec: 0.90 +[2025-04-25 02:06:02] (step=0086600) Train Loss: 6.3346, Train Steps/Sec: 0.90 +[2025-04-25 02:06:30] (step=0086625) Train Loss: 6.3373, Train Steps/Sec: 0.91 +[2025-04-25 02:06:57] (step=0086650) Train Loss: 6.3279, Train Steps/Sec: 0.90 +[2025-04-25 02:07:25] (step=0086675) Train Loss: 6.3092, Train Steps/Sec: 0.90 +[2025-04-25 02:07:53] (step=0086700) Train Loss: 6.2794, Train Steps/Sec: 0.90 +[2025-04-25 02:08:21] (step=0086725) Train Loss: 6.2821, Train Steps/Sec: 0.90 +[2025-04-25 02:08:49] (step=0086750) Train Loss: 6.2718, Train Steps/Sec: 0.90 +[2025-04-25 02:09:22] (step=0086775) Train Loss: 6.3062, Train Steps/Sec: 0.75 +[2025-04-25 02:09:50] (step=0086800) Train Loss: 6.3488, Train Steps/Sec: 0.90 +[2025-04-25 02:10:17] (step=0086825) Train Loss: 6.3040, Train Steps/Sec: 0.91 +[2025-04-25 02:10:45] (step=0086850) Train Loss: 6.2927, Train Steps/Sec: 0.91 +[2025-04-25 02:11:13] (step=0086875) Train Loss: 6.3427, Train Steps/Sec: 0.90 +[2025-04-25 02:11:40] (step=0086900) Train Loss: 6.3092, Train Steps/Sec: 0.90 +[2025-04-25 02:12:08] (step=0086925) Train Loss: 6.2936, Train Steps/Sec: 0.90 +[2025-04-25 02:12:36] (step=0086950) Train Loss: 6.3117, Train Steps/Sec: 0.91 +[2025-04-25 02:13:04] (step=0086975) Train Loss: 6.2978, Train Steps/Sec: 0.90 +[2025-04-25 02:13:31] (step=0087000) Train Loss: 6.3165, Train Steps/Sec: 0.90 +[2025-04-25 02:13:59] (step=0087025) Train Loss: 6.2955, Train Steps/Sec: 0.90 +[2025-04-25 02:14:27] (step=0087050) Train Loss: 6.2856, Train Steps/Sec: 0.90 +[2025-04-25 02:14:55] (step=0087075) Train Loss: 6.2954, Train Steps/Sec: 0.90 +[2025-04-25 02:15:22] (step=0087100) Train Loss: 6.2920, Train Steps/Sec: 0.90 +[2025-04-25 02:15:50] (step=0087125) Train Loss: 6.2945, Train Steps/Sec: 0.91 +[2025-04-25 02:16:18] (step=0087150) Train Loss: 6.3409, Train Steps/Sec: 0.90 +[2025-04-25 02:16:45] (step=0087175) Train Loss: 6.2625, Train Steps/Sec: 0.90 +[2025-04-25 02:17:13] (step=0087200) Train Loss: 6.2747, Train Steps/Sec: 0.90 +[2025-04-25 02:17:41] (step=0087225) Train Loss: 6.2830, Train Steps/Sec: 0.90 +[2025-04-25 02:18:09] (step=0087250) Train Loss: 6.2774, Train Steps/Sec: 0.91 +[2025-04-25 02:18:36] (step=0087275) Train Loss: 6.3041, Train Steps/Sec: 0.90 +[2025-04-25 02:19:04] (step=0087300) Train Loss: 6.3082, Train Steps/Sec: 0.90 +[2025-04-25 02:19:32] (step=0087325) Train Loss: 6.3364, Train Steps/Sec: 0.90 +[2025-04-25 02:19:59] (step=0087350) Train Loss: 6.2994, Train Steps/Sec: 0.90 +[2025-04-25 02:20:27] (step=0087375) Train Loss: 6.2704, Train Steps/Sec: 0.90 +[2025-04-25 02:20:55] (step=0087400) Train Loss: 6.2622, Train Steps/Sec: 0.90 +[2025-04-25 02:21:23] (step=0087425) Train Loss: 6.3095, Train Steps/Sec: 0.90 +[2025-04-25 02:21:50] (step=0087450) Train Loss: 6.2854, Train Steps/Sec: 0.91 +[2025-04-25 02:22:18] (step=0087475) Train Loss: 6.2772, Train Steps/Sec: 0.90 +[2025-04-25 02:22:46] (step=0087500) Train Loss: 6.3088, Train Steps/Sec: 0.90 +[2025-04-25 02:23:13] (step=0087525) Train Loss: 6.3019, Train Steps/Sec: 0.90 +[2025-04-25 02:23:41] (step=0087550) Train Loss: 6.3003, Train Steps/Sec: 0.90 +[2025-04-25 02:24:09] (step=0087575) Train Loss: 6.2610, Train Steps/Sec: 0.91 +[2025-04-25 02:24:37] (step=0087600) Train Loss: 6.2639, Train Steps/Sec: 0.90 +[2025-04-25 02:25:04] (step=0087625) Train Loss: 6.3034, Train Steps/Sec: 0.90 +[2025-04-25 02:25:32] (step=0087650) Train Loss: 6.2550, Train Steps/Sec: 0.90 +[2025-04-25 02:26:00] (step=0087675) Train Loss: 6.2987, Train Steps/Sec: 0.90 +[2025-04-25 02:26:28] (step=0087700) Train Loss: 6.2837, Train Steps/Sec: 0.90 +[2025-04-25 02:26:56] (step=0087725) Train Loss: 6.3282, Train Steps/Sec: 0.90 +[2025-04-25 02:27:23] (step=0087750) Train Loss: 6.2780, Train Steps/Sec: 0.90 +[2025-04-25 02:27:51] (step=0087775) Train Loss: 6.3237, Train Steps/Sec: 0.90 +[2025-04-25 02:28:19] (step=0087800) Train Loss: 6.3002, Train Steps/Sec: 0.90 +[2025-04-25 02:28:47] (step=0087825) Train Loss: 6.2739, Train Steps/Sec: 0.90 +[2025-04-25 02:29:14] (step=0087850) Train Loss: 6.3164, Train Steps/Sec: 0.90 +[2025-04-25 02:29:42] (step=0087875) Train Loss: 6.2999, Train Steps/Sec: 0.91 +[2025-04-25 02:30:10] (step=0087900) Train Loss: 6.3084, Train Steps/Sec: 0.90 +[2025-04-25 02:30:37] (step=0087925) Train Loss: 6.3223, Train Steps/Sec: 0.90 +[2025-04-25 02:31:05] (step=0087950) Train Loss: 6.2876, Train Steps/Sec: 0.91 +[2025-04-25 02:31:32] (step=0087975) Train Loss: 6.3445, Train Steps/Sec: 0.90 +[2025-04-25 02:32:00] (step=0088000) Train Loss: 6.3103, Train Steps/Sec: 0.90 +[2025-04-25 02:32:00] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-25 02:32:00] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-25 02:32:00] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 02:36:45] Finish Eval in 88000 steps... +[2025-04-25 02:37:07] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0088000.pt +[2025-04-25 02:37:09] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0086000.pt +[2025-04-25 02:37:37] (step=0088025) Train Loss: 6.3040, Train Steps/Sec: 0.07 +[2025-04-25 02:38:05] (step=0088050) Train Loss: 6.3146, Train Steps/Sec: 0.90 +[2025-04-25 02:38:32] (step=0088075) Train Loss: 6.2700, Train Steps/Sec: 0.91 +[2025-04-25 02:39:00] (step=0088100) Train Loss: 6.2805, Train Steps/Sec: 0.89 +[2025-04-25 02:39:28] (step=0088125) Train Loss: 6.3108, Train Steps/Sec: 0.91 +[2025-04-25 02:39:55] (step=0088150) Train Loss: 6.3120, Train Steps/Sec: 0.91 +[2025-04-25 02:40:23] (step=0088175) Train Loss: 6.3094, Train Steps/Sec: 0.90 +[2025-04-25 02:40:51] (step=0088200) Train Loss: 6.2554, Train Steps/Sec: 0.89 +[2025-04-25 02:41:19] (step=0088225) Train Loss: 6.2714, Train Steps/Sec: 0.90 +[2025-04-25 02:41:47] (step=0088250) Train Loss: 6.3072, Train Steps/Sec: 0.90 +[2025-04-25 02:42:14] (step=0088275) Train Loss: 6.3092, Train Steps/Sec: 0.91 +[2025-04-25 02:42:42] (step=0088300) Train Loss: 6.2877, Train Steps/Sec: 0.90 +[2025-04-25 02:43:10] (step=0088325) Train Loss: 6.2977, Train Steps/Sec: 0.91 +[2025-04-25 02:43:37] (step=0088350) Train Loss: 6.2850, Train Steps/Sec: 0.90 +[2025-04-25 02:44:05] (step=0088375) Train Loss: 6.3244, Train Steps/Sec: 0.90 +[2025-04-25 02:44:33] (step=0088400) Train Loss: 6.2553, Train Steps/Sec: 0.90 +[2025-04-25 02:45:01] (step=0088425) Train Loss: 6.3050, Train Steps/Sec: 0.90 +[2025-04-25 02:45:28] (step=0088450) Train Loss: 6.3403, Train Steps/Sec: 0.90 +[2025-04-25 02:45:56] (step=0088475) Train Loss: 6.2941, Train Steps/Sec: 0.90 +[2025-04-25 02:46:24] (step=0088500) Train Loss: 6.3056, Train Steps/Sec: 0.90 +[2025-04-25 02:46:51] (step=0088525) Train Loss: 6.3169, Train Steps/Sec: 0.91 +[2025-04-25 02:47:19] (step=0088550) Train Loss: 6.2624, Train Steps/Sec: 0.90 +[2025-04-25 02:47:47] (step=0088575) Train Loss: 6.3011, Train Steps/Sec: 0.90 +[2025-04-25 02:48:14] (step=0088600) Train Loss: 6.3055, Train Steps/Sec: 0.90 +[2025-04-25 02:48:42] (step=0088625) Train Loss: 6.3058, Train Steps/Sec: 0.90 +[2025-04-25 02:49:10] (step=0088650) Train Loss: 6.2882, Train Steps/Sec: 0.90 +[2025-04-25 02:49:38] (step=0088675) Train Loss: 6.2599, Train Steps/Sec: 0.90 +[2025-04-25 02:50:05] (step=0088700) Train Loss: 6.3065, Train Steps/Sec: 0.90 +[2025-04-25 02:50:33] (step=0088725) Train Loss: 6.3309, Train Steps/Sec: 0.90 +[2025-04-25 02:51:01] (step=0088750) Train Loss: 6.3137, Train Steps/Sec: 0.90 +[2025-04-25 02:51:29] (step=0088775) Train Loss: 6.2965, Train Steps/Sec: 0.90 +[2025-04-25 02:51:56] (step=0088800) Train Loss: 6.2905, Train Steps/Sec: 0.90 +[2025-04-25 02:52:24] (step=0088825) Train Loss: 6.2581, Train Steps/Sec: 0.90 +[2025-04-25 02:52:52] (step=0088850) Train Loss: 6.3276, Train Steps/Sec: 0.90 +[2025-04-25 02:53:20] (step=0088875) Train Loss: 6.3172, Train Steps/Sec: 0.90 +[2025-04-25 02:53:47] (step=0088900) Train Loss: 6.2845, Train Steps/Sec: 0.90 +[2025-04-25 02:54:15] (step=0088925) Train Loss: 6.2641, Train Steps/Sec: 0.90 +[2025-04-25 02:54:48] (step=0088950) Train Loss: 6.3168, Train Steps/Sec: 0.77 +[2025-04-25 02:55:15] (step=0088975) Train Loss: 6.2634, Train Steps/Sec: 0.91 +[2025-04-25 02:55:43] (step=0089000) Train Loss: 6.3062, Train Steps/Sec: 0.90 +[2025-04-25 02:56:16] (step=0089025) Train Loss: 6.3064, Train Steps/Sec: 0.77 +[2025-04-25 02:56:44] (step=0089050) Train Loss: 6.2868, Train Steps/Sec: 0.90 +[2025-04-25 02:57:16] (step=0089075) Train Loss: 6.2907, Train Steps/Sec: 0.76 +[2025-04-25 02:57:44] (step=0089100) Train Loss: 6.3028, Train Steps/Sec: 0.90 +[2025-04-25 02:58:12] (step=0089125) Train Loss: 6.3534, Train Steps/Sec: 0.90 +[2025-04-25 02:58:40] (step=0089150) Train Loss: 6.2347, Train Steps/Sec: 0.91 +[2025-04-25 02:59:07] (step=0089175) Train Loss: 6.2856, Train Steps/Sec: 0.90 +[2025-04-25 02:59:35] (step=0089200) Train Loss: 6.3176, Train Steps/Sec: 0.90 +[2025-04-25 03:00:03] (step=0089225) Train Loss: 6.2587, Train Steps/Sec: 0.91 +[2025-04-25 03:00:31] (step=0089250) Train Loss: 6.2923, Train Steps/Sec: 0.90 +[2025-04-25 03:00:58] (step=0089275) Train Loss: 6.2808, Train Steps/Sec: 0.90 +[2025-04-25 03:01:26] (step=0089300) Train Loss: 6.3330, Train Steps/Sec: 0.90 +[2025-04-25 03:01:59] (step=0089325) Train Loss: 6.3319, Train Steps/Sec: 0.77 +[2025-04-25 03:02:26] (step=0089350) Train Loss: 6.3387, Train Steps/Sec: 0.90 +[2025-04-25 03:02:54] (step=0089375) Train Loss: 6.2629, Train Steps/Sec: 0.91 +[2025-04-25 03:03:27] (step=0089400) Train Loss: 6.3081, Train Steps/Sec: 0.76 +[2025-04-25 03:03:55] (step=0089425) Train Loss: 6.3291, Train Steps/Sec: 0.90 +[2025-04-25 03:04:22] (step=0089450) Train Loss: 6.3304, Train Steps/Sec: 0.91 +[2025-04-25 03:04:55] (step=0089475) Train Loss: 6.3759, Train Steps/Sec: 0.76 +[2025-04-25 03:05:23] (step=0089500) Train Loss: 6.2596, Train Steps/Sec: 0.90 +[2025-04-25 03:05:51] (step=0089525) Train Loss: 6.2852, Train Steps/Sec: 0.91 +[2025-04-25 03:06:18] (step=0089550) Train Loss: 6.3186, Train Steps/Sec: 0.90 +[2025-04-25 03:06:46] (step=0089575) Train Loss: 6.2623, Train Steps/Sec: 0.91 +[2025-04-25 03:07:14] (step=0089600) Train Loss: 6.3035, Train Steps/Sec: 0.90 +[2025-04-25 03:07:41] (step=0089625) Train Loss: 6.3085, Train Steps/Sec: 0.91 +[2025-04-25 03:08:09] (step=0089650) Train Loss: 6.3141, Train Steps/Sec: 0.90 +[2025-04-25 03:08:37] (step=0089675) Train Loss: 6.2930, Train Steps/Sec: 0.90 +[2025-04-25 03:09:05] (step=0089700) Train Loss: 6.3078, Train Steps/Sec: 0.90 +[2025-04-25 03:09:32] (step=0089725) Train Loss: 6.3394, Train Steps/Sec: 0.90 +[2025-04-25 03:10:00] (step=0089750) Train Loss: 6.3161, Train Steps/Sec: 0.90 +[2025-04-25 03:10:28] (step=0089775) Train Loss: 6.3034, Train Steps/Sec: 0.91 +[2025-04-25 03:10:55] (step=0089800) Train Loss: 6.3170, Train Steps/Sec: 0.90 +[2025-04-25 03:11:23] (step=0089825) Train Loss: 6.3449, Train Steps/Sec: 0.91 +[2025-04-25 03:11:51] (step=0089850) Train Loss: 6.2610, Train Steps/Sec: 0.90 +[2025-04-25 03:12:18] (step=0089875) Train Loss: 6.3280, Train Steps/Sec: 0.90 +[2025-04-25 03:12:46] (step=0089900) Train Loss: 6.2787, Train Steps/Sec: 0.90 +[2025-04-25 03:13:14] (step=0089925) Train Loss: 6.3293, Train Steps/Sec: 0.90 +[2025-04-25 03:13:41] (step=0089950) Train Loss: 6.3107, Train Steps/Sec: 0.90 +[2025-04-25 03:14:09] (step=0089975) Train Loss: 6.3437, Train Steps/Sec: 0.90 +[2025-04-25 03:14:37] (step=0090000) Train Loss: 6.2817, Train Steps/Sec: 0.90 +[2025-04-25 03:14:37] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-25 03:14:37] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-25 03:14:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 03:19:24] Finish Eval in 90000 steps... +[2025-04-25 03:19:45] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0090000.pt +[2025-04-25 03:19:47] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0088000.pt +[2025-04-25 03:20:15] (step=0090025) Train Loss: 6.2824, Train Steps/Sec: 0.07 +[2025-04-25 03:20:43] (step=0090050) Train Loss: 6.2990, Train Steps/Sec: 0.90 +[2025-04-25 03:21:11] (step=0090075) Train Loss: 6.2982, Train Steps/Sec: 0.90 +[2025-04-25 03:21:39] (step=0090100) Train Loss: 6.2876, Train Steps/Sec: 0.90 +[2025-04-25 03:22:07] (step=0090125) Train Loss: 6.2717, Train Steps/Sec: 0.90 +[2025-04-25 03:22:34] (step=0090150) Train Loss: 6.3336, Train Steps/Sec: 0.90 +[2025-04-25 03:23:02] (step=0090175) Train Loss: 6.2825, Train Steps/Sec: 0.91 +[2025-04-25 03:23:30] (step=0090200) Train Loss: 6.2849, Train Steps/Sec: 0.90 +[2025-04-25 03:23:57] (step=0090225) Train Loss: 6.2645, Train Steps/Sec: 0.90 +[2025-04-25 03:24:25] (step=0090250) Train Loss: 6.2930, Train Steps/Sec: 0.90 +[2025-04-25 03:24:53] (step=0090275) Train Loss: 6.2673, Train Steps/Sec: 0.90 +[2025-04-25 03:25:21] (step=0090300) Train Loss: 6.2552, Train Steps/Sec: 0.90 +[2025-04-25 03:25:48] (step=0090325) Train Loss: 6.3100, Train Steps/Sec: 0.91 +[2025-04-25 03:26:16] (step=0090350) Train Loss: 6.3062, Train Steps/Sec: 0.91 +[2025-04-25 03:26:44] (step=0090375) Train Loss: 6.2920, Train Steps/Sec: 0.90 +[2025-04-25 03:27:11] (step=0090400) Train Loss: 6.3121, Train Steps/Sec: 0.90 +[2025-04-25 03:27:39] (step=0090425) Train Loss: 6.2936, Train Steps/Sec: 0.91 +[2025-04-25 03:28:06] (step=0090450) Train Loss: 6.3431, Train Steps/Sec: 0.91 +[2025-04-25 03:28:34] (step=0090475) Train Loss: 6.3088, Train Steps/Sec: 0.90 +[2025-04-25 03:29:02] (step=0090500) Train Loss: 6.2637, Train Steps/Sec: 0.90 +[2025-04-25 03:29:30] (step=0090525) Train Loss: 6.3165, Train Steps/Sec: 0.90 +[2025-04-25 03:29:57] (step=0090550) Train Loss: 6.3132, Train Steps/Sec: 0.90 +[2025-04-25 03:30:25] (step=0090575) Train Loss: 6.2973, Train Steps/Sec: 0.90 +[2025-04-25 03:30:53] (step=0090600) Train Loss: 6.3208, Train Steps/Sec: 0.90 +[2025-04-25 03:31:21] (step=0090625) Train Loss: 6.2988, Train Steps/Sec: 0.90 +[2025-04-25 03:31:48] (step=0090650) Train Loss: 6.3348, Train Steps/Sec: 0.90 +[2025-04-25 03:32:16] (step=0090675) Train Loss: 6.2510, Train Steps/Sec: 0.90 +[2025-04-25 03:32:44] (step=0090700) Train Loss: 6.3162, Train Steps/Sec: 0.90 +[2025-04-25 03:33:12] (step=0090725) Train Loss: 6.3292, Train Steps/Sec: 0.90 +[2025-04-25 03:33:39] (step=0090750) Train Loss: 6.2796, Train Steps/Sec: 0.90 +[2025-04-25 03:34:07] (step=0090775) Train Loss: 6.3029, Train Steps/Sec: 0.90 +[2025-04-25 03:34:35] (step=0090800) Train Loss: 6.2559, Train Steps/Sec: 0.90 +[2025-04-25 03:35:02] (step=0090825) Train Loss: 6.2947, Train Steps/Sec: 0.91 +[2025-04-25 03:35:30] (step=0090850) Train Loss: 6.2756, Train Steps/Sec: 0.90 +[2025-04-25 03:35:58] (step=0090875) Train Loss: 6.3191, Train Steps/Sec: 0.90 +[2025-04-25 03:36:26] (step=0090900) Train Loss: 6.2892, Train Steps/Sec: 0.90 +[2025-04-25 03:36:53] (step=0090925) Train Loss: 6.3120, Train Steps/Sec: 0.90 +[2025-04-25 03:37:21] (step=0090950) Train Loss: 6.3054, Train Steps/Sec: 0.90 +[2025-04-25 03:37:49] (step=0090975) Train Loss: 6.3085, Train Steps/Sec: 0.90 +[2025-04-25 03:38:17] (step=0091000) Train Loss: 6.2819, Train Steps/Sec: 0.90 +[2025-04-25 03:38:44] (step=0091025) Train Loss: 6.3684, Train Steps/Sec: 0.90 +[2025-04-25 03:39:12] (step=0091050) Train Loss: 6.3019, Train Steps/Sec: 0.91 +[2025-04-25 03:39:40] (step=0091075) Train Loss: 6.3376, Train Steps/Sec: 0.90 +[2025-04-25 03:40:07] (step=0091100) Train Loss: 6.2707, Train Steps/Sec: 0.90 +[2025-04-25 03:40:35] (step=0091125) Train Loss: 6.2692, Train Steps/Sec: 0.90 +[2025-04-25 03:41:03] (step=0091150) Train Loss: 6.3318, Train Steps/Sec: 0.90 +[2025-04-25 03:41:31] (step=0091175) Train Loss: 6.3087, Train Steps/Sec: 0.90 +[2025-04-25 03:41:58] (step=0091200) Train Loss: 6.3040, Train Steps/Sec: 0.90 +[2025-04-25 03:42:26] (step=0091225) Train Loss: 6.2815, Train Steps/Sec: 0.90 +[2025-04-25 03:42:54] (step=0091250) Train Loss: 6.2902, Train Steps/Sec: 0.90 +[2025-04-25 03:43:22] (step=0091275) Train Loss: 6.3302, Train Steps/Sec: 0.91 +[2025-04-25 03:43:49] (step=0091300) Train Loss: 6.3278, Train Steps/Sec: 0.90 +[2025-04-25 03:44:17] (step=0091325) Train Loss: 6.3469, Train Steps/Sec: 0.91 +[2025-04-25 03:44:45] (step=0091350) Train Loss: 6.2995, Train Steps/Sec: 0.91 +[2025-04-25 03:45:12] (step=0091375) Train Loss: 6.3196, Train Steps/Sec: 0.90 +[2025-04-25 03:45:40] (step=0091400) Train Loss: 6.2833, Train Steps/Sec: 0.90 +[2025-04-25 03:46:08] (step=0091425) Train Loss: 6.2967, Train Steps/Sec: 0.90 +[2025-04-25 03:46:35] (step=0091450) Train Loss: 6.2939, Train Steps/Sec: 0.90 +[2025-04-25 03:47:03] (step=0091475) Train Loss: 6.3149, Train Steps/Sec: 0.90 +[2025-04-25 03:47:31] (step=0091500) Train Loss: 6.3162, Train Steps/Sec: 0.90 +[2025-04-25 03:47:59] (step=0091525) Train Loss: 6.2371, Train Steps/Sec: 0.90 +[2025-04-25 03:48:26] (step=0091550) Train Loss: 6.2834, Train Steps/Sec: 0.90 +[2025-04-25 03:48:54] (step=0091575) Train Loss: 6.3168, Train Steps/Sec: 0.91 +[2025-04-25 03:49:22] (step=0091600) Train Loss: 6.2638, Train Steps/Sec: 0.90 +[2025-04-25 03:49:49] (step=0091625) Train Loss: 6.3145, Train Steps/Sec: 0.90 +[2025-04-25 03:50:17] (step=0091650) Train Loss: 6.2659, Train Steps/Sec: 0.90 +[2025-04-25 03:50:45] (step=0091675) Train Loss: 6.3200, Train Steps/Sec: 0.90 +[2025-04-25 03:51:13] (step=0091700) Train Loss: 6.2762, Train Steps/Sec: 0.90 +[2025-04-25 03:51:41] (step=0091725) Train Loss: 6.3236, Train Steps/Sec: 0.89 +[2025-04-25 03:52:08] (step=0091750) Train Loss: 6.3527, Train Steps/Sec: 0.90 +[2025-04-25 03:52:36] (step=0091775) Train Loss: 6.3383, Train Steps/Sec: 0.90 +[2025-04-25 03:53:04] (step=0091800) Train Loss: 6.3187, Train Steps/Sec: 0.89 +[2025-04-25 03:53:32] (step=0091825) Train Loss: 6.3321, Train Steps/Sec: 0.90 +[2025-04-25 03:53:43] Done! diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/debug-internal.log b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..873f0d256eef5ca67150ae6cdee7899d676d46f1 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-04-24T14:31:23.684611405Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug-core.log"} +{"time":"2025-04-24T14:31:23.891641414Z","level":"INFO","msg":"created new stream","id":"yx27lyn0"} +{"time":"2025-04-24T14:31:23.891678633Z","level":"INFO","msg":"stream: started","id":"yx27lyn0"} +{"time":"2025-04-24T14:31:23.891700099Z","level":"INFO","msg":"writer: Do: started","stream_id":"yx27lyn0"} +{"time":"2025-04-24T14:31:23.891715989Z","level":"INFO","msg":"sender: started","stream_id":"yx27lyn0"} +{"time":"2025-04-24T14:31:23.891910858Z","level":"INFO","msg":"handler: started","stream_id":"yx27lyn0"} +{"time":"2025-04-24T14:31:24.428314594Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-25T03:53:49.294061502Z","level":"INFO","msg":"stream: closing","id":"yx27lyn0"} +{"time":"2025-04-25T03:53:49.294102644Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-25T03:53:49.295055957Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-25T03:53:50.416600201Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-25T03:53:50.691812308Z","level":"INFO","msg":"handler: closed","stream_id":"yx27lyn0"} +{"time":"2025-04-25T03:53:50.691857783Z","level":"INFO","msg":"writer: Close: closed","stream_id":"yx27lyn0"} +{"time":"2025-04-25T03:53:50.691886326Z","level":"INFO","msg":"sender: closed","stream_id":"yx27lyn0"} +{"time":"2025-04-25T03:53:50.691949154Z","level":"INFO","msg":"stream: closed","id":"yx27lyn0"} diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/debug.log b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..95150326adf7028bd69677201422ae4808d94263 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/debug.log @@ -0,0 +1,23 @@ +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_setup.py:_flush():67] Configure stats pid to 2575513 +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug.log +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug-internal.log +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_init.py:init():761] calling init triggers +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_trained_raw_cleaned.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0056000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 5e-05, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_val_raw_cleaned.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/instructblip-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 200, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'instructblip', 'do_recovery': True, 'no_replace': False, 'resume': True, 'dreambench_eval': False, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': None, 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_init.py:init():784] starting backend +2025-04-24 14:31:23,677 INFO MainThread:2575513 [wandb_init.py:init():788] sending inform_init request +2025-04-24 14:31:23,682 INFO MainThread:2575513 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-24 14:31:23,682 INFO MainThread:2575513 [wandb_init.py:init():798] backend started and connected +2025-04-24 14:31:23,685 INFO MainThread:2575513 [wandb_init.py:init():891] updated telemetry +2025-04-24 14:31:23,690 INFO MainThread:2575513 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-24 14:31:24,425 INFO MainThread:2575513 [wandb_init.py:init():990] starting run threads in backend +2025-04-24 14:31:24,511 INFO MainThread:2575513 [wandb_run.py:_console_start():2375] atexit reg +2025-04-24 14:31:24,511 INFO MainThread:2575513 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-24 14:31:24,512 INFO MainThread:2575513 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-24 14:31:24,513 INFO MainThread:2575513 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-24 14:31:24,514 INFO MainThread:2575513 [wandb_init.py:init():1032] run started, returning control to user process +2025-04-25 03:53:49,293 INFO MsgRouterThr:2575513 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/files/output.log b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c850378d11cca02b40e0199af46981bc90252e48 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/files/output.log @@ -0,0 +1,2713 @@ +[2025-04-23 17:22:10] Training for 2 epochs... +[2025-04-23 17:22:10] Beginning epoch 0... + 0%| | 0/45917 [00:00 +tokenizer length after expend 32102 +tokenizer length before expend 32102 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/3 [00:00 +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.35s/it] +[2025-04-23 19:34:15] Finish Eval in 4000 steps...████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 58.84s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 19:34:38] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0004000.pt +[2025-04-23 19:34:40] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0002000.pt + 9%|█████████▎ | 4024/45917 [2:12:56<13:19:57, 1.15s/it][2025-04-23 19:35:08] (step=0004025) Train Loss: 6.4821, Train Steps/Sec: 0.07 + 9%|█████████▎ | 4049/45917 [2:13:24<12:50:46, 1.10s/it][2025-04-23 19:35:36] (step=0004050) Train Loss: 6.4506, Train Steps/Sec: 0.90 + 9%|█████████▍ | 4074/45917 [2:13:52<12:46:08, 1.10s/it][2025-04-23 19:36:03] (step=0004075) Train Loss: 6.4983, Train Steps/Sec: 0.90 + 9%|█████████▍ | 4099/45917 [2:14:20<12:46:04, 1.10s/it][2025-04-23 19:36:31] (step=0004100) Train Loss: 6.4911, Train Steps/Sec: 0.90 + 9%|█████████▌ | 4124/45917 [2:14:47<13:00:01, 1.12s/it][2025-04-23 19:36:59] (step=0004125) Train Loss: 6.4749, Train Steps/Sec: 0.91 + 9%|█████████▌ | 4149/45917 [2:15:15<12:46:41, 1.10s/it][2025-04-23 19:37:26] (step=0004150) Train Loss: 6.4284, Train Steps/Sec: 0.90 + 9%|█████████▋ | 4174/45917 [2:15:43<12:49:00, 1.11s/it][2025-04-23 19:37:54] (step=0004175) Train Loss: 6.5049, Train Steps/Sec: 0.90 + 9%|█████████▋ | 4199/45917 [2:16:11<12:49:17, 1.11s/it][2025-04-23 19:38:22] (step=0004200) Train Loss: 6.5086, Train Steps/Sec: 0.90 + 9%|█████████▊ | 4224/45917 [2:16:39<13:27:38, 1.16s/it][2025-04-23 19:38:50] (step=0004225) Train Loss: 6.4990, Train Steps/Sec: 0.89 + 9%|█████████▊ | 4249/45917 [2:17:07<13:01:36, 1.13s/it][2025-04-23 19:39:18] (step=0004250) Train Loss: 6.5159, Train Steps/Sec: 0.90 + 9%|█████████▊ | 4274/45917 [2:17:34<12:54:30, 1.12s/it][2025-04-23 19:39:46] (step=0004275) Train Loss: 6.4676, Train Steps/Sec: 0.90 + 9%|█████████▉ | 4299/45917 [2:18:02<12:35:24, 1.09s/it][2025-04-23 19:40:14] (step=0004300) Train Loss: 6.4395, Train Steps/Sec: 0.89 + 9%|█████████▉ | 4324/45917 [2:18:30<13:02:32, 1.13s/it][2025-04-23 19:40:41] (step=0004325) Train Loss: 6.5199, Train Steps/Sec: 0.90 + 9%|██████████ | 4349/45917 [2:18:58<12:55:40, 1.12s/it][2025-04-23 19:41:09] (step=0004350) Train Loss: 6.4711, Train Steps/Sec: 0.90 + 10%|██████████ | 4374/45917 [2:19:26<12:40:52, 1.10s/it][2025-04-23 19:41:37] (step=0004375) Train Loss: 6.5152, Train Steps/Sec: 0.90 + 10%|██████████▏ | 4399/45917 [2:19:53<12:44:46, 1.11s/it][2025-04-23 19:42:05] (step=0004400) Train Loss: 6.5036, Train Steps/Sec: 0.90 + 10%|██████████▏ | 4424/45917 [2:20:21<12:50:54, 1.11s/it][2025-04-23 19:42:32] (step=0004425) Train Loss: 6.4928, Train Steps/Sec: 0.91 + 10%|██████████▎ | 4449/45917 [2:20:49<12:52:46, 1.12s/it][2025-04-23 19:43:00] (step=0004450) Train Loss: 6.4760, Train Steps/Sec: 0.90 + 10%|██████████▎ | 4474/45917 [2:21:17<12:41:41, 1.10s/it][2025-04-23 19:43:28] (step=0004475) Train Loss: 6.5068, Train Steps/Sec: 0.90 + 10%|██████████▍ | 4499/45917 [2:21:44<12:43:12, 1.11s/it][2025-04-23 19:43:56] (step=0004500) Train Loss: 6.5035, Train Steps/Sec: 0.90 + 10%|██████████▍ | 4524/45917 [2:22:12<12:54:39, 1.12s/it][2025-04-23 19:44:23] (step=0004525) Train Loss: 6.4963, Train Steps/Sec: 0.90 + 10%|██████████▌ | 4549/45917 [2:22:40<12:41:49, 1.10s/it][2025-04-23 19:44:51] (step=0004550) Train Loss: 6.4936, Train Steps/Sec: 0.90 + 10%|██████████▌ | 4574/45917 [2:23:08<12:42:03, 1.11s/it][2025-04-23 19:45:19] (step=0004575) Train Loss: 6.4659, Train Steps/Sec: 0.90 + 10%|██████████▌ | 4599/45917 [2:23:35<12:34:04, 1.10s/it][2025-04-23 19:45:47] (step=0004600) Train Loss: 6.4722, Train Steps/Sec: 0.90 + 10%|██████████▋ | 4624/45917 [2:24:03<12:52:18, 1.12s/it][2025-04-23 19:46:14] (step=0004625) Train Loss: 6.4550, Train Steps/Sec: 0.91 + 10%|██████████▋ | 4649/45917 [2:24:31<12:47:01, 1.12s/it][2025-04-23 19:46:42] (step=0004650) Train Loss: 6.4851, Train Steps/Sec: 0.90 + 10%|██████████▊ | 4674/45917 [2:24:59<12:36:48, 1.10s/it][2025-04-23 19:47:10] (step=0004675) Train Loss: 6.4471, Train Steps/Sec: 0.90 + 10%|██████████▊ | 4699/45917 [2:25:26<12:32:22, 1.10s/it][2025-04-23 19:47:38] (step=0004700) Train Loss: 6.4892, Train Steps/Sec: 0.90 + 10%|██████████▉ | 4724/45917 [2:25:54<12:53:22, 1.13s/it][2025-04-23 19:48:06] (step=0004725) Train Loss: 6.4649, Train Steps/Sec: 0.90 + 10%|██████████▉ | 4749/45917 [2:26:22<12:46:30, 1.12s/it][2025-04-23 19:48:33] (step=0004750) Train Loss: 6.5252, Train Steps/Sec: 0.90 + 10%|███████████ | 4774/45917 [2:26:50<12:41:52, 1.11s/it][2025-04-23 19:49:01] (step=0004775) Train Loss: 6.5150, Train Steps/Sec: 0.90 + 10%|███████████ | 4799/45917 [2:27:18<12:29:17, 1.09s/it][2025-04-23 19:49:29] (step=0004800) Train Loss: 6.4127, Train Steps/Sec: 0.90 + 11%|███████████▏ | 4824/45917 [2:27:45<12:48:41, 1.12s/it][2025-04-23 19:49:57] (step=0004825) Train Loss: 6.4442, Train Steps/Sec: 0.90 + 11%|███████████▏ | 4849/45917 [2:28:13<12:38:03, 1.11s/it][2025-04-23 19:50:24] (step=0004850) Train Loss: 6.5087, Train Steps/Sec: 0.90 + 11%|███████████▏ | 4870/45917 [2:28:37<13:16:07, 1.16s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 11%|███████████▎ | 4874/45917 [2:28:41<12:50:53, 1.13s/it][2025-04-23 19:50:52] (step=0004875) Train Loss: 6.5230, Train Steps/Sec: 0.89 + 11%|███████████▎ | 4899/45917 [2:29:09<12:30:36, 1.10s/it][2025-04-23 19:51:20] (step=0004900) Train Loss: 6.4496, Train Steps/Sec: 0.90 + 11%|███████████▎ | 4924/45917 [2:29:37<12:49:11, 1.13s/it][2025-04-23 19:51:48] (step=0004925) Train Loss: 6.4608, Train Steps/Sec: 0.90 + 11%|███████████▍ | 4949/45917 [2:30:05<12:48:30, 1.13s/it][2025-04-23 19:52:16] (step=0004950) Train Loss: 6.4698, Train Steps/Sec: 0.89 + 11%|███████████▍ | 4974/45917 [2:30:33<12:33:40, 1.10s/it][2025-04-23 19:52:44] (step=0004975) Train Loss: 6.4607, Train Steps/Sec: 0.90 + 11%|███████████▌ | 4999/45917 [2:31:00<12:28:13, 1.10s/it][2025-04-23 19:53:12] (step=0005000) Train Loss: 6.4582, Train Steps/Sec: 0.90 + 11%|███████████▌ | 5024/45917 [2:31:28<12:52:33, 1.13s/it][2025-04-23 19:53:39] (step=0005025) Train Loss: 6.5307, Train Steps/Sec: 0.90 + 11%|███████████▋ | 5049/45917 [2:31:56<12:40:10, 1.12s/it][2025-04-23 19:54:07] (step=0005050) Train Loss: 6.4422, Train Steps/Sec: 0.90 + 11%|███████████▋ | 5074/45917 [2:32:24<12:30:26, 1.10s/it][2025-04-23 19:54:35] (step=0005075) Train Loss: 6.5171, Train Steps/Sec: 0.90 + 11%|███████████▊ | 5099/45917 [2:32:51<12:24:05, 1.09s/it][2025-04-23 19:55:03] (step=0005100) Train Loss: 6.4786, Train Steps/Sec: 0.90 + 11%|███████████▊ | 5124/45917 [2:33:19<12:43:32, 1.12s/it][2025-04-23 19:55:30] (step=0005125) Train Loss: 6.4761, Train Steps/Sec: 0.90 + 11%|███████████▉ | 5149/45917 [2:33:47<12:35:45, 1.11s/it][2025-04-23 19:55:58] (step=0005150) Train Loss: 6.4695, Train Steps/Sec: 0.90 + 11%|███████████▉ | 5174/45917 [2:34:15<12:29:01, 1.10s/it][2025-04-23 19:56:26] (step=0005175) Train Loss: 6.5127, Train Steps/Sec: 0.90 + 11%|████████████ | 5199/45917 [2:34:43<12:26:46, 1.10s/it][2025-04-23 19:56:54] (step=0005200) Train Loss: 6.4941, Train Steps/Sec: 0.89 + 11%|████████████ | 5224/45917 [2:35:10<12:47:07, 1.13s/it][2025-04-23 19:57:22] (step=0005225) Train Loss: 6.4551, Train Steps/Sec: 0.90 + 11%|████████████ | 5249/45917 [2:35:38<12:34:28, 1.11s/it][2025-04-23 19:57:50] (step=0005250) Train Loss: 6.5093, Train Steps/Sec: 0.90 + 11%|████████████▏ | 5274/45917 [2:36:06<12:23:21, 1.10s/it][2025-04-23 19:58:17] (step=0005275) Train Loss: 6.4366, Train Steps/Sec: 0.91 + 12%|████████████▏ | 5299/45917 [2:36:34<12:25:30, 1.10s/it][2025-04-23 19:58:45] (step=0005300) Train Loss: 6.4456, Train Steps/Sec: 0.90 + 12%|████████████▎ | 5324/45917 [2:37:02<12:39:34, 1.12s/it][2025-04-23 19:59:13] (step=0005325) Train Loss: 6.4591, Train Steps/Sec: 0.90 + 12%|████████████▎ | 5349/45917 [2:37:29<12:38:02, 1.12s/it][2025-04-23 19:59:41] (step=0005350) Train Loss: 6.4774, Train Steps/Sec: 0.90 + 12%|████████████▍ | 5374/45917 [2:37:57<12:21:19, 1.10s/it][2025-04-23 20:00:08] (step=0005375) Train Loss: 6.4567, Train Steps/Sec: 0.90 + 12%|████████████▍ | 5399/45917 [2:38:25<12:19:21, 1.09s/it][2025-04-23 20:00:36] (step=0005400) Train Loss: 6.4879, Train Steps/Sec: 0.90 + 12%|████████████▌ | 5424/45917 [2:38:53<12:43:28, 1.13s/it][2025-04-23 20:01:04] (step=0005425) Train Loss: 6.4692, Train Steps/Sec: 0.90 + 12%|████████████▌ | 5449/45917 [2:39:21<12:25:03, 1.10s/it][2025-04-23 20:01:32] (step=0005450) Train Loss: 6.4326, Train Steps/Sec: 0.90 + 12%|████████████▋ | 5474/45917 [2:39:48<12:29:00, 1.11s/it][2025-04-23 20:01:59] (step=0005475) Train Loss: 6.4946, Train Steps/Sec: 0.90 + 12%|████████████▋ | 5499/45917 [2:40:16<12:17:06, 1.09s/it][2025-04-23 20:02:27] (step=0005500) Train Loss: 6.4410, Train Steps/Sec: 0.90 + 12%|████████████▊ | 5524/45917 [2:40:44<12:39:13, 1.13s/it][2025-04-23 20:02:55] (step=0005525) Train Loss: 6.5103, Train Steps/Sec: 0.90 + 12%|████████████▊ | 5549/45917 [2:41:12<12:26:19, 1.11s/it][2025-04-23 20:03:23] (step=0005550) Train Loss: 6.4812, Train Steps/Sec: 0.90 + 12%|████████████▊ | 5574/45917 [2:41:40<12:20:14, 1.10s/it][2025-04-23 20:03:51] (step=0005575) Train Loss: 6.4761, Train Steps/Sec: 0.90 + 12%|████████████▉ | 5599/45917 [2:42:08<12:24:02, 1.11s/it][2025-04-23 20:04:19] (step=0005600) Train Loss: 6.5083, Train Steps/Sec: 0.89 + 12%|████████████▉ | 5624/45917 [2:42:35<12:34:51, 1.12s/it][2025-04-23 20:04:47] (step=0005625) Train Loss: 6.4665, Train Steps/Sec: 0.90 + 12%|█████████████ | 5649/45917 [2:43:03<12:21:36, 1.11s/it][2025-04-23 20:05:14] (step=0005650) Train Loss: 6.4507, Train Steps/Sec: 0.90 + 12%|█████████████ | 5674/45917 [2:43:31<12:15:45, 1.10s/it][2025-04-23 20:05:42] (step=0005675) Train Loss: 6.4444, Train Steps/Sec: 0.91 + 12%|█████████████▏ | 5699/45917 [2:43:58<12:15:43, 1.10s/it][2025-04-23 20:06:10] (step=0005700) Train Loss: 6.4792, Train Steps/Sec: 0.90 + 12%|█████████████▏ | 5724/45917 [2:44:26<12:42:11, 1.14s/it][2025-04-23 20:06:38] (step=0005725) Train Loss: 6.5288, Train Steps/Sec: 0.90 + 13%|█████████████▎ | 5749/45917 [2:44:54<12:26:29, 1.12s/it][2025-04-23 20:07:05] (step=0005750) Train Loss: 6.4635, Train Steps/Sec: 0.90 + 13%|█████████████▎ | 5774/45917 [2:45:22<12:23:32, 1.11s/it][2025-04-23 20:07:33] (step=0005775) Train Loss: 6.4467, Train Steps/Sec: 0.90 + 13%|█████████████▍ | 5799/45917 [2:45:50<12:16:35, 1.10s/it][2025-04-23 20:08:01] (step=0005800) Train Loss: 6.4837, Train Steps/Sec: 0.90 + 13%|█████████████▍ | 5824/45917 [2:46:18<12:38:41, 1.14s/it][2025-04-23 20:08:29] (step=0005825) Train Loss: 6.4768, Train Steps/Sec: 0.90 + 13%|█████████████▌ | 5849/45917 [2:46:46<12:28:21, 1.12s/it][2025-04-23 20:08:57] (step=0005850) Train Loss: 6.4928, Train Steps/Sec: 0.89 + 13%|█████████████▌ | 5874/45917 [2:47:13<12:10:54, 1.10s/it][2025-04-23 20:09:25] (step=0005875) Train Loss: 6.4833, Train Steps/Sec: 0.90 + 13%|█████████████▌ | 5899/45917 [2:47:41<12:08:27, 1.09s/it][2025-04-23 20:09:53] (step=0005900) Train Loss: 6.4333, Train Steps/Sec: 0.90 + 13%|█████████████▋ | 5924/45917 [2:48:09<12:49:26, 1.15s/it][2025-04-23 20:10:20] (step=0005925) Train Loss: 6.4376, Train Steps/Sec: 0.90 + 13%|█████████████▋ | 5949/45917 [2:48:37<12:19:38, 1.11s/it][2025-04-23 20:10:48] (step=0005950) Train Loss: 6.4496, Train Steps/Sec: 0.90 + 13%|█████████████▊ | 5974/45917 [2:49:05<12:18:45, 1.11s/it][2025-04-23 20:11:16] (step=0005975) Train Loss: 6.4208, Train Steps/Sec: 0.90 + 13%|█████████████▊ | 5999/45917 [2:49:33<12:06:58, 1.09s/it][2025-04-23 20:11:44] (step=0006000) Train Loss: 6.4739, Train Steps/Sec: 0.90 +[2025-04-23 20:11:44] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-23 20:11:44] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-23 20:11:44] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/3 [00:00 +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.72s/it] +[2025-04-23 20:58:48] Finish Eval in 8000 steps...████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.27s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 20:59:10] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0008000.pt +[2025-04-23 20:59:13] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0006000.pt + 17%|██████████████████▌ | 8024/45917 [3:37:29<12:06:03, 1.15s/it][2025-04-23 20:59:41] (step=0008025) Train Loss: 6.4248, Train Steps/Sec: 0.07 + 18%|██████████████████▌ | 8049/45917 [3:37:57<11:46:00, 1.12s/it][2025-04-23 21:00:08] (step=0008050) Train Loss: 6.4251, Train Steps/Sec: 0.90 + 18%|██████████████████▋ | 8074/45917 [3:38:25<11:35:08, 1.10s/it][2025-04-23 21:00:36] (step=0008075) Train Loss: 6.4434, Train Steps/Sec: 0.90 + 18%|██████████████████▋ | 8099/45917 [3:38:53<11:32:37, 1.10s/it][2025-04-23 21:01:04] (step=0008100) Train Loss: 6.4616, Train Steps/Sec: 0.89 + 18%|██████████████████▊ | 8124/45917 [3:39:21<11:51:14, 1.13s/it][2025-04-23 21:01:32] (step=0008125) Train Loss: 6.4380, Train Steps/Sec: 0.90 + 18%|██████████████████▊ | 8149/45917 [3:39:49<11:36:10, 1.11s/it][2025-04-23 21:02:00] (step=0008150) Train Loss: 6.4496, Train Steps/Sec: 0.90 + 18%|██████████████████▊ | 8174/45917 [3:40:16<11:31:20, 1.10s/it][2025-04-23 21:02:28] (step=0008175) Train Loss: 6.4301, Train Steps/Sec: 0.90 + 18%|██████████████████▉ | 8199/45917 [3:40:44<11:35:40, 1.11s/it][2025-04-23 21:02:55] (step=0008200) Train Loss: 6.4888, Train Steps/Sec: 0.90 + 18%|██████████████████▉ | 8224/45917 [3:41:12<11:52:30, 1.13s/it][2025-04-23 21:03:23] (step=0008225) Train Loss: 6.5032, Train Steps/Sec: 0.90 + 18%|███████████████████ | 8249/45917 [3:41:40<11:37:41, 1.11s/it][2025-04-23 21:03:51] (step=0008250) Train Loss: 6.4503, Train Steps/Sec: 0.90 + 18%|███████████████████ | 8274/45917 [3:42:07<11:32:10, 1.10s/it][2025-04-23 21:04:19] (step=0008275) Train Loss: 6.4530, Train Steps/Sec: 0.90 + 18%|███████████████████▏ | 8299/45917 [3:42:35<11:28:51, 1.10s/it][2025-04-23 21:04:47] (step=0008300) Train Loss: 6.4442, Train Steps/Sec: 0.89 + 18%|███████████████████▏ | 8324/45917 [3:43:03<11:45:39, 1.13s/it][2025-04-23 21:05:14] (step=0008325) Train Loss: 6.4576, Train Steps/Sec: 0.90 + 18%|███████████████████▎ | 8349/45917 [3:43:31<11:37:42, 1.11s/it][2025-04-23 21:05:42] (step=0008350) Train Loss: 6.4454, Train Steps/Sec: 0.90 + 18%|███████████████████▎ | 8374/45917 [3:44:06<11:29:28, 1.10s/it][2025-04-23 21:06:17] (step=0008375) Train Loss: 6.4698, Train Steps/Sec: 0.71 + 18%|███████████████████▍ | 8399/45917 [3:44:34<11:28:54, 1.10s/it][2025-04-23 21:06:45] (step=0008400) Train Loss: 6.4878, Train Steps/Sec: 0.90 + 18%|███████████████████▍ | 8424/45917 [3:45:02<11:46:43, 1.13s/it][2025-04-23 21:07:13] (step=0008425) Train Loss: 6.4464, Train Steps/Sec: 0.90 + 18%|███████████████████▌ | 8449/45917 [3:45:36<11:32:17, 1.11s/it][2025-04-23 21:07:48] (step=0008450) Train Loss: 6.4501, Train Steps/Sec: 0.72 + 18%|███████████████████▌ | 8474/45917 [3:46:04<11:32:16, 1.11s/it][2025-04-23 21:08:15] (step=0008475) Train Loss: 6.4563, Train Steps/Sec: 0.90 + 19%|███████████████████▌ | 8499/45917 [3:46:32<11:26:36, 1.10s/it][2025-04-23 21:08:43] (step=0008500) Train Loss: 6.4500, Train Steps/Sec: 0.90 + 19%|███████████████████▋ | 8524/45917 [3:47:00<11:40:58, 1.12s/it][2025-04-23 21:09:11] (step=0008525) Train Loss: 6.4876, Train Steps/Sec: 0.90 + 19%|███████████████████▋ | 8549/45917 [3:47:28<11:40:14, 1.12s/it][2025-04-23 21:09:39] (step=0008550) Train Loss: 6.4647, Train Steps/Sec: 0.90 + 19%|███████████████████▊ | 8574/45917 [3:48:02<11:46:55, 1.14s/it][2025-04-23 21:10:13] (step=0008575) Train Loss: 6.4290, Train Steps/Sec: 0.74 + 19%|███████████████████▊ | 8599/45917 [3:48:29<11:22:30, 1.10s/it][2025-04-23 21:10:41] (step=0008600) Train Loss: 6.4232, Train Steps/Sec: 0.90 + 19%|███████████████████▉ | 8624/45917 [3:48:57<11:38:56, 1.12s/it][2025-04-23 21:11:08] (step=0008625) Train Loss: 6.4363, Train Steps/Sec: 0.90 + 19%|███████████████████▉ | 8649/45917 [3:49:31<11:28:41, 1.11s/it][2025-04-23 21:11:42] (step=0008650) Train Loss: 6.4327, Train Steps/Sec: 0.74 + 19%|████████████████████ | 8674/45917 [3:49:59<11:22:37, 1.10s/it][2025-04-23 21:12:10] (step=0008675) Train Loss: 6.4607, Train Steps/Sec: 0.90 + 19%|████████████████████ | 8699/45917 [3:50:27<11:22:30, 1.10s/it][2025-04-23 21:12:38] (step=0008700) Train Loss: 6.4589, Train Steps/Sec: 0.90 + 19%|████████████████████▏ | 8724/45917 [3:50:55<11:41:00, 1.13s/it][2025-04-23 21:13:06] (step=0008725) Train Loss: 6.4549, Train Steps/Sec: 0.90 + 19%|████████████████████▏ | 8749/45917 [3:51:22<11:25:47, 1.11s/it][2025-04-23 21:13:34] (step=0008750) Train Loss: 6.4573, Train Steps/Sec: 0.90 + 19%|████████████████████▎ | 8774/45917 [3:51:57<30:52:39, 2.99s/it][2025-04-23 21:14:08] (step=0008775) Train Loss: 6.4125, Train Steps/Sec: 0.73 + 19%|████████████████████▎ | 8799/45917 [3:52:24<11:23:19, 1.10s/it][2025-04-23 21:14:36] (step=0008800) Train Loss: 6.4308, Train Steps/Sec: 0.90 + 19%|████████████████████▎ | 8824/45917 [3:52:52<11:31:49, 1.12s/it][2025-04-23 21:15:03] (step=0008825) Train Loss: 6.4777, Train Steps/Sec: 0.90 + 19%|████████████████████▍ | 8849/45917 [3:53:27<11:26:29, 1.11s/it][2025-04-23 21:15:38] (step=0008850) Train Loss: 6.4373, Train Steps/Sec: 0.73 + 19%|████████████████████▍ | 8874/45917 [3:54:01<11:29:17, 1.12s/it][2025-04-23 21:16:12] (step=0008875) Train Loss: 6.4390, Train Steps/Sec: 0.73 + 19%|████████████████████▌ | 8899/45917 [3:54:29<11:22:53, 1.11s/it][2025-04-23 21:16:40] (step=0008900) Train Loss: 6.4551, Train Steps/Sec: 0.89 + 19%|████████████████████▌ | 8924/45917 [3:54:57<11:37:56, 1.13s/it][2025-04-23 21:17:08] (step=0008925) Train Loss: 6.4404, Train Steps/Sec: 0.90 + 19%|████████████████████▋ | 8949/45917 [3:55:24<11:24:46, 1.11s/it][2025-04-23 21:17:36] (step=0008950) Train Loss: 6.4419, Train Steps/Sec: 0.90 + 20%|████████████████████▋ | 8974/45917 [3:55:52<11:23:21, 1.11s/it][2025-04-23 21:18:04] (step=0008975) Train Loss: 6.4235, Train Steps/Sec: 0.90 + 20%|████████████████████▊ | 8999/45917 [3:56:20<11:19:22, 1.10s/it][2025-04-23 21:18:32] (step=0009000) Train Loss: 6.4440, Train Steps/Sec: 0.89 + 20%|████████████████████▊ | 9024/45917 [3:56:48<11:35:36, 1.13s/it][2025-04-23 21:18:59] (step=0009025) Train Loss: 6.4199, Train Steps/Sec: 0.90 + 20%|████████████████████▉ | 9049/45917 [3:57:16<11:30:07, 1.12s/it][2025-04-23 21:19:27] (step=0009050) Train Loss: 6.4346, Train Steps/Sec: 0.89 + 20%|████████████████████▉ | 9074/45917 [3:57:44<11:18:09, 1.10s/it][2025-04-23 21:19:55] (step=0009075) Train Loss: 6.4456, Train Steps/Sec: 0.90 + 20%|█████████████████████ | 9099/45917 [3:58:12<11:20:19, 1.11s/it][2025-04-23 21:20:23] (step=0009100) Train Loss: 6.4926, Train Steps/Sec: 0.89 + 20%|█████████████████████ | 9124/45917 [3:58:40<11:36:20, 1.14s/it][2025-04-23 21:20:51] (step=0009125) Train Loss: 6.5172, Train Steps/Sec: 0.90 + 20%|█████████████████████ | 9149/45917 [3:59:08<11:28:43, 1.12s/it][2025-04-23 21:21:19] (step=0009150) Train Loss: 6.4566, Train Steps/Sec: 0.90 + 20%|█████████████████████▏ | 9174/45917 [3:59:36<11:20:04, 1.11s/it][2025-04-23 21:21:47] (step=0009175) Train Loss: 6.4749, Train Steps/Sec: 0.90 + 20%|█████████████████████▏ | 9199/45917 [4:00:04<11:19:15, 1.11s/it][2025-04-23 21:22:15] (step=0009200) Train Loss: 6.4522, Train Steps/Sec: 0.89 + 20%|█████████████████████▎ | 9224/45917 [4:00:32<11:29:46, 1.13s/it][2025-04-23 21:22:43] (step=0009225) Train Loss: 6.4828, Train Steps/Sec: 0.90 + 20%|█████████████████████▎ | 9249/45917 [4:01:00<11:21:52, 1.12s/it][2025-04-23 21:23:11] (step=0009250) Train Loss: 6.4241, Train Steps/Sec: 0.90 + 20%|█████████████████████▍ | 9274/45917 [4:01:27<11:16:55, 1.11s/it][2025-04-23 21:23:39] (step=0009275) Train Loss: 6.4430, Train Steps/Sec: 0.90 + 20%|█████████████████████▍ | 9299/45917 [4:01:55<11:10:28, 1.10s/it][2025-04-23 21:24:07] (step=0009300) Train Loss: 6.4278, Train Steps/Sec: 0.89 + 20%|█████████████████████▌ | 9324/45917 [4:02:23<11:26:12, 1.13s/it][2025-04-23 21:24:34] (step=0009325) Train Loss: 6.4146, Train Steps/Sec: 0.90 + 20%|█████████████████████▌ | 9349/45917 [4:02:51<11:17:49, 1.11s/it][2025-04-23 21:25:02] (step=0009350) Train Loss: 6.4488, Train Steps/Sec: 0.90 + 20%|█████████████████████▋ | 9374/45917 [4:03:19<11:18:11, 1.11s/it][2025-04-23 21:25:30] (step=0009375) Train Loss: 6.4719, Train Steps/Sec: 0.90 + 20%|█████████████████████▋ | 9399/45917 [4:03:47<11:11:08, 1.10s/it][2025-04-23 21:25:58] (step=0009400) Train Loss: 6.4608, Train Steps/Sec: 0.89 + 21%|█████████████████████▊ | 9424/45917 [4:04:15<11:35:21, 1.14s/it][2025-04-23 21:26:26] (step=0009425) Train Loss: 6.4350, Train Steps/Sec: 0.90 + 21%|█████████████████████▊ | 9449/45917 [4:04:43<11:13:40, 1.11s/it][2025-04-23 21:26:54] (step=0009450) Train Loss: 6.4207, Train Steps/Sec: 0.90 + 21%|█████████████████████▊ | 9474/45917 [4:05:10<11:18:15, 1.12s/it][2025-04-23 21:27:22] (step=0009475) Train Loss: 6.4294, Train Steps/Sec: 0.90 + 21%|█████████████████████▉ | 9499/45917 [4:05:38<11:07:05, 1.10s/it][2025-04-23 21:27:50] (step=0009500) Train Loss: 6.4375, Train Steps/Sec: 0.89 + 21%|█████████████████████▉ | 9524/45917 [4:06:06<11:24:15, 1.13s/it][2025-04-23 21:28:18] (step=0009525) Train Loss: 6.4235, Train Steps/Sec: 0.90 + 21%|██████████████████████ | 9549/45917 [4:06:34<11:16:55, 1.12s/it][2025-04-23 21:28:45] (step=0009550) Train Loss: 6.4458, Train Steps/Sec: 0.90 + 21%|██████████████████████ | 9574/45917 [4:07:02<11:07:20, 1.10s/it][2025-04-23 21:29:13] (step=0009575) Train Loss: 6.4444, Train Steps/Sec: 0.90 + 21%|██████████████████████▏ | 9599/45917 [4:07:30<11:14:20, 1.11s/it][2025-04-23 21:29:41] (step=0009600) Train Loss: 6.4368, Train Steps/Sec: 0.89 + 21%|██████████████████████▏ | 9624/45917 [4:07:58<11:26:48, 1.14s/it][2025-04-23 21:30:09] (step=0009625) Train Loss: 6.4544, Train Steps/Sec: 0.90 + 21%|██████████████████████▎ | 9649/45917 [4:08:26<11:08:42, 1.11s/it][2025-04-23 21:30:37] (step=0009650) Train Loss: 6.4521, Train Steps/Sec: 0.90 + 21%|██████████████████████▎ | 9674/45917 [4:08:54<11:09:47, 1.11s/it][2025-04-23 21:31:05] (step=0009675) Train Loss: 6.4401, Train Steps/Sec: 0.90 + 21%|██████████████████████▍ | 9699/45917 [4:09:21<11:01:00, 1.10s/it][2025-04-23 21:31:33] (step=0009700) Train Loss: 6.4293, Train Steps/Sec: 0.90 + 21%|██████████████████████▍ | 9724/45917 [4:09:49<11:24:28, 1.13s/it][2025-04-23 21:32:00] (step=0009725) Train Loss: 6.4543, Train Steps/Sec: 0.90 + 21%|██████████████████████▌ | 9749/45917 [4:10:17<11:02:50, 1.10s/it][2025-04-23 21:32:28] (step=0009750) Train Loss: 6.4358, Train Steps/Sec: 0.90 + 21%|██████████████████████▌ | 9774/45917 [4:10:44<11:01:28, 1.10s/it][2025-04-23 21:32:56] (step=0009775) Train Loss: 6.4567, Train Steps/Sec: 0.91 + 21%|██████████████████████▌ | 9799/45917 [4:11:12<10:58:19, 1.09s/it][2025-04-23 21:33:24] (step=0009800) Train Loss: 6.4604, Train Steps/Sec: 0.90 + 21%|██████████████████████▋ | 9824/45917 [4:11:40<11:14:22, 1.12s/it][2025-04-23 21:33:51] (step=0009825) Train Loss: 6.4254, Train Steps/Sec: 0.90 + 21%|██████████████████████▋ | 9849/45917 [4:12:08<11:06:39, 1.11s/it][2025-04-23 21:34:19] (step=0009850) Train Loss: 6.4850, Train Steps/Sec: 0.90 + 22%|██████████████████████▊ | 9874/45917 [4:12:35<10:59:16, 1.10s/it][2025-04-23 21:34:47] (step=0009875) Train Loss: 6.4407, Train Steps/Sec: 0.91 + 22%|██████████████████████▊ | 9899/45917 [4:13:03<10:56:34, 1.09s/it][2025-04-23 21:35:14] (step=0009900) Train Loss: 6.4377, Train Steps/Sec: 0.90 + 22%|██████████████████████▉ | 9924/45917 [4:13:31<11:19:39, 1.13s/it][2025-04-23 21:35:42] (step=0009925) Train Loss: 6.4605, Train Steps/Sec: 0.90 + 22%|██████████████████████▉ | 9949/45917 [4:13:59<11:04:21, 1.11s/it][2025-04-23 21:36:10] (step=0009950) Train Loss: 6.4899, Train Steps/Sec: 0.90 + 22%|███████████████████████ | 9974/45917 [4:14:26<10:59:39, 1.10s/it][2025-04-23 21:36:38] (step=0009975) Train Loss: 6.4463, Train Steps/Sec: 0.90 + 22%|███████████████████████ | 9999/45917 [4:14:54<10:56:52, 1.10s/it][2025-04-23 21:37:05] (step=0010000) Train Loss: 6.4262, Train Steps/Sec: 0.90 +[2025-04-23 21:37:06] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-23 21:37:06] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-23 21:37:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:59<00:00, 59.69s/it] +[2025-04-23 21:41:58] Finish Eval in 10000 steps...███████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.22s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 21:42:20] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0010000.pt +[2025-04-23 21:42:22] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0008000.pt + 22%|██████████████████████▉ | 10024/45917 [4:20:39<11:30:15, 1.15s/it][2025-04-23 21:42:50] (step=0010025) Train Loss: 6.3961, Train Steps/Sec: 0.07 + 22%|██████████████████████▉ | 10049/45917 [4:21:07<11:08:13, 1.12s/it][2025-04-23 21:43:18] (step=0010050) Train Loss: 6.3897, Train Steps/Sec: 0.90 + 22%|███████████████████████ | 10074/45917 [4:21:34<11:03:46, 1.11s/it][2025-04-23 21:43:46] (step=0010075) Train Loss: 6.4658, Train Steps/Sec: 0.90 + 22%|███████████████████████ | 10099/45917 [4:22:02<10:48:37, 1.09s/it][2025-04-23 21:44:13] (step=0010100) Train Loss: 6.4608, Train Steps/Sec: 0.90 + 22%|███████████████████████▏ | 10124/45917 [4:22:30<11:10:18, 1.12s/it][2025-04-23 21:44:41] (step=0010125) Train Loss: 6.4668, Train Steps/Sec: 0.90 + 22%|███████████████████████▏ | 10149/45917 [4:22:57<10:57:19, 1.10s/it][2025-04-23 21:45:09] (step=0010150) Train Loss: 6.4177, Train Steps/Sec: 0.91 + 22%|███████████████████████▎ | 10174/45917 [4:23:25<10:54:59, 1.10s/it][2025-04-23 21:45:36] (step=0010175) Train Loss: 6.4570, Train Steps/Sec: 0.91 + 22%|███████████████████████▎ | 10199/45917 [4:23:53<10:53:53, 1.10s/it][2025-04-23 21:46:04] (step=0010200) Train Loss: 6.4354, Train Steps/Sec: 0.90 + 22%|███████████████████████▍ | 10224/45917 [4:24:21<11:08:02, 1.12s/it][2025-04-23 21:46:32] (step=0010225) Train Loss: 6.4665, Train Steps/Sec: 0.90 + 22%|███████████████████████▍ | 10249/45917 [4:24:48<10:57:38, 1.11s/it][2025-04-23 21:47:00] (step=0010250) Train Loss: 6.4109, Train Steps/Sec: 0.90 + 22%|███████████████████████▍ | 10274/45917 [4:25:16<10:52:00, 1.10s/it][2025-04-23 21:47:27] (step=0010275) Train Loss: 6.4387, Train Steps/Sec: 0.91 + 22%|███████████████████████▌ | 10299/45917 [4:25:44<10:50:15, 1.10s/it][2025-04-23 21:47:55] (step=0010300) Train Loss: 6.4496, Train Steps/Sec: 0.90 + 22%|███████████████████████▌ | 10324/45917 [4:26:11<11:15:20, 1.14s/it][2025-04-23 21:48:23] (step=0010325) Train Loss: 6.4380, Train Steps/Sec: 0.90 + 23%|███████████████████████▋ | 10349/45917 [4:26:39<11:00:16, 1.11s/it][2025-04-23 21:48:50] (step=0010350) Train Loss: 6.4560, Train Steps/Sec: 0.90 + 23%|███████████████████████▋ | 10374/45917 [4:27:07<10:57:36, 1.11s/it][2025-04-23 21:49:18] (step=0010375) Train Loss: 6.4102, Train Steps/Sec: 0.90 + 23%|███████████████████████▊ | 10399/45917 [4:27:35<10:50:46, 1.10s/it][2025-04-23 21:49:46] (step=0010400) Train Loss: 6.4119, Train Steps/Sec: 0.90 + 23%|███████████████████████▊ | 10424/45917 [4:28:03<11:07:42, 1.13s/it][2025-04-23 21:50:14] (step=0010425) Train Loss: 6.4174, Train Steps/Sec: 0.90 + 23%|███████████████████████▉ | 10449/45917 [4:28:30<11:00:35, 1.12s/it][2025-04-23 21:50:42] (step=0010450) Train Loss: 6.4109, Train Steps/Sec: 0.90 + 23%|███████████████████████▉ | 10474/45917 [4:28:58<10:52:20, 1.10s/it][2025-04-23 21:51:09] (step=0010475) Train Loss: 6.4257, Train Steps/Sec: 0.90 + 23%|████████████████████████ | 10499/45917 [4:29:26<10:50:33, 1.10s/it][2025-04-23 21:51:37] (step=0010500) Train Loss: 6.4565, Train Steps/Sec: 0.90 + 23%|████████████████████████ | 10524/45917 [4:29:54<11:03:19, 1.12s/it][2025-04-23 21:52:05] (step=0010525) Train Loss: 6.4087, Train Steps/Sec: 0.90 + 23%|████████████████████████ | 10549/45917 [4:30:21<10:54:37, 1.11s/it][2025-04-23 21:52:33] (step=0010550) Train Loss: 6.4642, Train Steps/Sec: 0.90 + 23%|████████████████████████▏ | 10574/45917 [4:30:49<10:50:58, 1.11s/it][2025-04-23 21:53:00] (step=0010575) Train Loss: 6.4116, Train Steps/Sec: 0.90 + 23%|████████████████████████▏ | 10599/45917 [4:31:17<10:46:45, 1.10s/it][2025-04-23 21:53:28] (step=0010600) Train Loss: 6.4150, Train Steps/Sec: 0.90 + 23%|████████████████████████▎ | 10624/45917 [4:31:45<10:59:17, 1.12s/it][2025-04-23 21:53:56] (step=0010625) Train Loss: 6.4201, Train Steps/Sec: 0.90 + 23%|████████████████████████▎ | 10649/45917 [4:32:13<10:59:11, 1.12s/it][2025-04-23 21:54:24] (step=0010650) Train Loss: 6.4654, Train Steps/Sec: 0.90 + 23%|████████████████████████▍ | 10674/45917 [4:32:40<10:49:38, 1.11s/it][2025-04-23 21:54:52] (step=0010675) Train Loss: 6.4918, Train Steps/Sec: 0.90 + 23%|████████████████████████▍ | 10699/45917 [4:33:08<10:46:46, 1.10s/it][2025-04-23 21:55:19] (step=0010700) Train Loss: 6.4641, Train Steps/Sec: 0.90 + 23%|████████████████████████▌ | 10724/45917 [4:33:36<10:59:33, 1.12s/it][2025-04-23 21:55:47] (step=0010725) Train Loss: 6.4371, Train Steps/Sec: 0.90 + 23%|████████████████████████▌ | 10749/45917 [4:34:04<10:55:32, 1.12s/it][2025-04-23 21:56:15] (step=0010750) Train Loss: 6.4364, Train Steps/Sec: 0.90 + 23%|████████████████████████▋ | 10774/45917 [4:34:32<10:43:59, 1.10s/it][2025-04-23 21:56:43] (step=0010775) Train Loss: 6.4022, Train Steps/Sec: 0.90 + 24%|████████████████████████▋ | 10799/45917 [4:34:59<10:38:45, 1.09s/it][2025-04-23 21:57:10] (step=0010800) Train Loss: 6.4472, Train Steps/Sec: 0.90 + 24%|████████████████████████▊ | 10824/45917 [4:35:27<10:57:24, 1.12s/it][2025-04-23 21:57:38] (step=0010825) Train Loss: 6.4175, Train Steps/Sec: 0.91 + 24%|████████████████████████▊ | 10849/45917 [4:35:55<10:46:23, 1.11s/it][2025-04-23 21:58:06] (step=0010850) Train Loss: 6.4669, Train Steps/Sec: 0.90 + 24%|████████████████████████▊ | 10874/45917 [4:36:22<10:45:22, 1.10s/it][2025-04-23 21:58:34] (step=0010875) Train Loss: 6.4620, Train Steps/Sec: 0.90 + 24%|████████████████████████▉ | 10899/45917 [4:36:50<10:43:08, 1.10s/it][2025-04-23 21:59:01] (step=0010900) Train Loss: 6.4592, Train Steps/Sec: 0.90 + 24%|████████████████████████▉ | 10924/45917 [4:37:18<10:55:21, 1.12s/it][2025-04-23 21:59:29] (step=0010925) Train Loss: 6.4259, Train Steps/Sec: 0.90 + 24%|█████████████████████████ | 10949/45917 [4:37:46<10:48:42, 1.11s/it][2025-04-23 21:59:57] (step=0010950) Train Loss: 6.3933, Train Steps/Sec: 0.90 + 24%|█████████████████████████ | 10974/45917 [4:38:13<10:39:02, 1.10s/it][2025-04-23 22:00:25] (step=0010975) Train Loss: 6.4324, Train Steps/Sec: 0.91 + 24%|█████████████████████████▏ | 10999/45917 [4:38:41<10:34:49, 1.09s/it][2025-04-23 22:00:52] (step=0011000) Train Loss: 6.4541, Train Steps/Sec: 0.90 + 24%|█████████████████████████▏ | 11024/45917 [4:39:09<10:53:33, 1.12s/it][2025-04-23 22:01:20] (step=0011025) Train Loss: 6.4370, Train Steps/Sec: 0.90 + 24%|█████████████████████████▎ | 11049/45917 [4:39:36<10:41:29, 1.10s/it][2025-04-23 22:01:48] (step=0011050) Train Loss: 6.4508, Train Steps/Sec: 0.90 + 24%|█████████████████████████▎ | 11074/45917 [4:40:04<10:38:10, 1.10s/it][2025-04-23 22:02:15] (step=0011075) Train Loss: 6.4807, Train Steps/Sec: 0.90 + 24%|█████████████████████████▍ | 11099/45917 [4:40:32<10:35:51, 1.10s/it][2025-04-23 22:02:43] (step=0011100) Train Loss: 6.4375, Train Steps/Sec: 0.90 + 24%|█████████████████████████▍ | 11124/45917 [4:41:00<10:57:04, 1.13s/it][2025-04-23 22:03:11] (step=0011125) Train Loss: 6.4635, Train Steps/Sec: 0.90 + 24%|█████████████████████████▍ | 11149/45917 [4:41:27<10:43:51, 1.11s/it][2025-04-23 22:03:39] (step=0011150) Train Loss: 6.4846, Train Steps/Sec: 0.90 + 24%|█████████████████████████▌ | 11174/45917 [4:41:55<10:37:54, 1.10s/it][2025-04-23 22:04:06] (step=0011175) Train Loss: 6.4593, Train Steps/Sec: 0.90 + 24%|█████████████████████████▌ | 11199/45917 [4:42:23<10:34:18, 1.10s/it][2025-04-23 22:04:34] (step=0011200) Train Loss: 6.4517, Train Steps/Sec: 0.90 + 24%|█████████████████████████▋ | 11224/45917 [4:42:51<10:51:34, 1.13s/it][2025-04-23 22:05:02] (step=0011225) Train Loss: 6.4171, Train Steps/Sec: 0.90 + 24%|█████████████████████████▋ | 11249/45917 [4:43:19<10:42:39, 1.11s/it][2025-04-23 22:05:30] (step=0011250) Train Loss: 6.4093, Train Steps/Sec: 0.90 + 25%|█████████████████████████▊ | 11274/45917 [4:43:46<10:36:01, 1.10s/it][2025-04-23 22:05:58] (step=0011275) Train Loss: 6.4669, Train Steps/Sec: 0.90 + 25%|█████████████████████████▊ | 11299/45917 [4:44:14<10:29:48, 1.09s/it][2025-04-23 22:06:25] (step=0011300) Train Loss: 6.4023, Train Steps/Sec: 0.90 + 25%|█████████████████████████▉ | 11324/45917 [4:44:42<10:53:06, 1.13s/it][2025-04-23 22:06:53] (step=0011325) Train Loss: 6.4576, Train Steps/Sec: 0.90 + 25%|█████████████████████████▉ | 11349/45917 [4:45:10<10:41:30, 1.11s/it][2025-04-23 22:07:21] (step=0011350) Train Loss: 6.4495, Train Steps/Sec: 0.90 + 25%|██████████████████████████ | 11374/45917 [4:45:37<10:32:50, 1.10s/it][2025-04-23 22:07:49] (step=0011375) Train Loss: 6.4126, Train Steps/Sec: 0.90 + 25%|██████████████████████████ | 11399/45917 [4:46:05<10:26:10, 1.09s/it][2025-04-23 22:08:17] (step=0011400) Train Loss: 6.4681, Train Steps/Sec: 0.90 + 25%|██████████████████████████ | 11424/45917 [4:46:33<10:43:34, 1.12s/it][2025-04-23 22:08:44] (step=0011425) Train Loss: 6.4348, Train Steps/Sec: 0.90 + 25%|██████████████████████████▏ | 11449/45917 [4:47:01<10:33:58, 1.10s/it][2025-04-23 22:09:12] (step=0011450) Train Loss: 6.4704, Train Steps/Sec: 0.90 + 25%|██████████████████████████▏ | 11474/45917 [4:47:28<10:28:34, 1.09s/it][2025-04-23 22:09:40] (step=0011475) Train Loss: 6.4272, Train Steps/Sec: 0.90 + 25%|██████████████████████████▎ | 11499/45917 [4:47:56<10:28:05, 1.09s/it][2025-04-23 22:10:07] (step=0011500) Train Loss: 6.3949, Train Steps/Sec: 0.90 + 25%|██████████████████████████▎ | 11524/45917 [4:48:24<10:51:00, 1.14s/it][2025-04-23 22:10:35] (step=0011525) Train Loss: 6.4682, Train Steps/Sec: 0.90 + 25%|██████████████████████████▍ | 11549/45917 [4:48:52<10:35:31, 1.11s/it][2025-04-23 22:11:03] (step=0011550) Train Loss: 6.4196, Train Steps/Sec: 0.90 + 25%|██████████████████████████▍ | 11574/45917 [4:49:19<10:33:45, 1.11s/it][2025-04-23 22:11:31] (step=0011575) Train Loss: 6.4006, Train Steps/Sec: 0.90 + 25%|██████████████████████████▌ | 11599/45917 [4:49:47<10:29:51, 1.10s/it][2025-04-23 22:11:58] (step=0011600) Train Loss: 6.4824, Train Steps/Sec: 0.90 + 25%|██████████████████████████▌ | 11624/45917 [4:50:15<10:42:29, 1.12s/it][2025-04-23 22:12:26] (step=0011625) Train Loss: 6.4321, Train Steps/Sec: 0.91 + 25%|██████████████████████████▋ | 11649/45917 [4:50:42<10:28:20, 1.10s/it][2025-04-23 22:12:54] (step=0011650) Train Loss: 6.4421, Train Steps/Sec: 0.90 + 25%|██████████████████████████▋ | 11674/45917 [4:51:10<10:27:44, 1.10s/it][2025-04-23 22:13:21] (step=0011675) Train Loss: 6.4381, Train Steps/Sec: 0.90 + 25%|██████████████████████████▊ | 11699/45917 [4:51:38<10:26:32, 1.10s/it][2025-04-23 22:13:49] (step=0011700) Train Loss: 6.4084, Train Steps/Sec: 0.90 + 26%|██████████████████████████▊ | 11724/45917 [4:52:06<10:37:56, 1.12s/it][2025-04-23 22:14:17] (step=0011725) Train Loss: 6.4538, Train Steps/Sec: 0.90 + 26%|██████████████████████████▊ | 11749/45917 [4:52:33<10:34:06, 1.11s/it][2025-04-23 22:14:45] (step=0011750) Train Loss: 6.4339, Train Steps/Sec: 0.90 + 26%|██████████████████████████▉ | 11774/45917 [4:53:01<10:26:04, 1.10s/it][2025-04-23 22:15:12] (step=0011775) Train Loss: 6.4305, Train Steps/Sec: 0.90 + 26%|██████████████████████████▉ | 11799/45917 [4:53:29<10:22:19, 1.09s/it][2025-04-23 22:15:40] (step=0011800) Train Loss: 6.4370, Train Steps/Sec: 0.90 + 26%|███████████████████████████ | 11824/45917 [4:53:57<10:39:43, 1.13s/it][2025-04-23 22:16:08] (step=0011825) Train Loss: 6.4429, Train Steps/Sec: 0.90 + 26%|███████████████████████████ | 11849/45917 [4:54:24<10:29:06, 1.11s/it][2025-04-23 22:16:36] (step=0011850) Train Loss: 6.4449, Train Steps/Sec: 0.90 + 26%|███████████████████████████▏ | 11874/45917 [4:54:52<10:23:46, 1.10s/it][2025-04-23 22:17:03] (step=0011875) Train Loss: 6.4320, Train Steps/Sec: 0.91 + 26%|███████████████████████████▏ | 11899/45917 [4:55:20<10:27:47, 1.11s/it][2025-04-23 22:17:31] (step=0011900) Train Loss: 6.4138, Train Steps/Sec: 0.90 + 26%|███████████████████████████▎ | 11924/45917 [4:55:47<10:35:29, 1.12s/it][2025-04-23 22:17:59] (step=0011925) Train Loss: 6.4247, Train Steps/Sec: 0.90 + 26%|███████████████████████████▎ | 11949/45917 [4:56:15<10:27:33, 1.11s/it][2025-04-23 22:18:26] (step=0011950) Train Loss: 6.4334, Train Steps/Sec: 0.91 + 26%|███████████████████████████▍ | 11974/45917 [4:56:43<10:17:53, 1.09s/it][2025-04-23 22:18:54] (step=0011975) Train Loss: 6.4632, Train Steps/Sec: 0.90 + 26%|███████████████████████████▍ | 11999/45917 [4:57:11<10:17:39, 1.09s/it][2025-04-23 22:19:22] (step=0012000) Train Loss: 6.4471, Train Steps/Sec: 0.90 +[2025-04-23 22:19:22] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-23 22:19:22] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-23 22:19:22] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 59.00s/it] +[2025-04-23 22:24:08] Finish Eval in 12000 steps...███████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.49s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 22:24:30] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0012000.pt +[2025-04-23 22:24:32] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0010000.pt + 26%|███████████████████████████▍ | 12024/45917 [5:02:49<10:49:37, 1.15s/it][2025-04-23 22:25:00] (step=0012025) Train Loss: 6.4706, Train Steps/Sec: 0.07 + 26%|███████████████████████████▌ | 12049/45917 [5:03:17<10:24:40, 1.11s/it][2025-04-23 22:25:28] (step=0012050) Train Loss: 6.4473, Train Steps/Sec: 0.90 + 26%|███████████████████████████▌ | 12074/45917 [5:03:45<10:27:23, 1.11s/it][2025-04-23 22:25:56] (step=0012075) Train Loss: 6.4296, Train Steps/Sec: 0.90 + 26%|███████████████████████████▋ | 12099/45917 [5:04:12<10:19:35, 1.10s/it][2025-04-23 22:26:23] (step=0012100) Train Loss: 6.4181, Train Steps/Sec: 0.90 + 26%|███████████████████████████▋ | 12124/45917 [5:04:40<10:33:02, 1.12s/it][2025-04-23 22:26:51] (step=0012125) Train Loss: 6.4683, Train Steps/Sec: 0.90 + 26%|███████████████████████████▊ | 12149/45917 [5:05:08<10:17:43, 1.10s/it][2025-04-23 22:27:19] (step=0012150) Train Loss: 6.4324, Train Steps/Sec: 0.90 + 27%|███████████████████████████▊ | 12174/45917 [5:05:35<10:17:01, 1.10s/it][2025-04-23 22:27:47] (step=0012175) Train Loss: 6.4293, Train Steps/Sec: 0.90 + 27%|███████████████████████████▉ | 12199/45917 [5:06:03<10:14:51, 1.09s/it][2025-04-23 22:28:14] (step=0012200) Train Loss: 6.4426, Train Steps/Sec: 0.90 + 27%|███████████████████████████▉ | 12224/45917 [5:06:31<10:45:51, 1.15s/it][2025-04-23 22:28:42] (step=0012225) Train Loss: 6.4509, Train Steps/Sec: 0.90 + 27%|████████████████████████████ | 12249/45917 [5:06:58<10:20:00, 1.10s/it][2025-04-23 22:29:10] (step=0012250) Train Loss: 6.4773, Train Steps/Sec: 0.90 + 27%|████████████████████████████ | 12274/45917 [5:07:26<10:21:57, 1.11s/it][2025-04-23 22:29:38] (step=0012275) Train Loss: 6.4673, Train Steps/Sec: 0.90 + 27%|████████████████████████████ | 12299/45917 [5:07:54<10:16:15, 1.10s/it][2025-04-23 22:30:05] (step=0012300) Train Loss: 6.4625, Train Steps/Sec: 0.90 + 27%|████████████████████████████▏ | 12324/45917 [5:08:22<10:29:17, 1.12s/it][2025-04-23 22:30:33] (step=0012325) Train Loss: 6.4937, Train Steps/Sec: 0.90 + 27%|████████████████████████████▏ | 12349/45917 [5:08:50<10:25:04, 1.12s/it][2025-04-23 22:31:01] (step=0012350) Train Loss: 6.3967, Train Steps/Sec: 0.90 + 27%|████████████████████████████▎ | 12374/45917 [5:09:17<10:15:56, 1.10s/it][2025-04-23 22:31:28] (step=0012375) Train Loss: 6.4433, Train Steps/Sec: 0.90 + 27%|████████████████████████████▎ | 12399/45917 [5:09:45<10:21:27, 1.11s/it][2025-04-23 22:31:56] (step=0012400) Train Loss: 6.4063, Train Steps/Sec: 0.90 + 27%|████████████████████████████▍ | 12424/45917 [5:10:13<10:23:04, 1.12s/it][2025-04-23 22:32:24] (step=0012425) Train Loss: 6.4166, Train Steps/Sec: 0.91 + 27%|████████████████████████████▍ | 12449/45917 [5:10:40<10:18:59, 1.11s/it][2025-04-23 22:32:52] (step=0012450) Train Loss: 6.4006, Train Steps/Sec: 0.90 + 27%|████████████████████████████▌ | 12474/45917 [5:11:09<10:17:31, 1.11s/it][2025-04-23 22:33:20] (step=0012475) Train Loss: 6.4148, Train Steps/Sec: 0.89 + 27%|████████████████████████████▌ | 12499/45917 [5:11:36<10:10:55, 1.10s/it][2025-04-23 22:33:48] (step=0012500) Train Loss: 6.3998, Train Steps/Sec: 0.90 + 27%|████████████████████████████▋ | 12524/45917 [5:12:04<10:22:47, 1.12s/it][2025-04-23 22:34:15] (step=0012525) Train Loss: 6.4625, Train Steps/Sec: 0.90 + 27%|████████████████████████████▋ | 12549/45917 [5:12:32<10:20:08, 1.12s/it][2025-04-23 22:34:43] (step=0012550) Train Loss: 6.4276, Train Steps/Sec: 0.90 + 27%|████████████████████████████▊ | 12574/45917 [5:13:00<10:18:34, 1.11s/it][2025-04-23 22:35:11] (step=0012575) Train Loss: 6.4242, Train Steps/Sec: 0.90 + 27%|████████████████████████████▊ | 12599/45917 [5:13:27<10:10:52, 1.10s/it][2025-04-23 22:35:39] (step=0012600) Train Loss: 6.4617, Train Steps/Sec: 0.90 + 27%|████████████████████████████▊ | 12624/45917 [5:13:55<10:25:23, 1.13s/it][2025-04-23 22:36:06] (step=0012625) Train Loss: 6.4029, Train Steps/Sec: 0.90 + 28%|████████████████████████████▉ | 12649/45917 [5:14:23<10:10:51, 1.10s/it][2025-04-23 22:36:34] (step=0012650) Train Loss: 6.4397, Train Steps/Sec: 0.90 + 28%|████████████████████████████▉ | 12674/45917 [5:14:50<10:12:47, 1.11s/it][2025-04-23 22:37:02] (step=0012675) Train Loss: 6.4139, Train Steps/Sec: 0.90 + 28%|█████████████████████████████ | 12699/45917 [5:15:18<10:07:04, 1.10s/it][2025-04-23 22:37:29] (step=0012700) Train Loss: 6.3923, Train Steps/Sec: 0.90 + 28%|█████████████████████████████ | 12724/45917 [5:15:46<10:24:03, 1.13s/it][2025-04-23 22:37:57] (step=0012725) Train Loss: 6.4095, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▏ | 12749/45917 [5:16:14<10:21:58, 1.13s/it][2025-04-23 22:38:25] (step=0012750) Train Loss: 6.3883, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▏ | 12774/45917 [5:16:41<10:08:38, 1.10s/it][2025-04-23 22:38:53] (step=0012775) Train Loss: 6.4245, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▎ | 12799/45917 [5:17:09<10:03:56, 1.09s/it][2025-04-23 22:39:20] (step=0012800) Train Loss: 6.4661, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▎ | 12824/45917 [5:17:37<10:24:03, 1.13s/it][2025-04-23 22:39:48] (step=0012825) Train Loss: 6.4464, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▍ | 12849/45917 [5:18:05<10:10:34, 1.11s/it][2025-04-23 22:40:16] (step=0012850) Train Loss: 6.4652, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▍ | 12874/45917 [5:18:33<10:07:22, 1.10s/it][2025-04-23 22:40:44] (step=0012875) Train Loss: 6.4564, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▍ | 12899/45917 [5:19:00<10:07:38, 1.10s/it][2025-04-23 22:41:12] (step=0012900) Train Loss: 6.4668, Train Steps/Sec: 0.89 + 28%|█████████████████████████████▌ | 12924/45917 [5:19:28<10:16:44, 1.12s/it][2025-04-23 22:41:39] (step=0012925) Train Loss: 6.3972, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▌ | 12949/45917 [5:19:56<10:06:48, 1.10s/it][2025-04-23 22:42:07] (step=0012950) Train Loss: 6.4310, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▋ | 12974/45917 [5:20:24<10:05:19, 1.10s/it][2025-04-23 22:42:35] (step=0012975) Train Loss: 6.4099, Train Steps/Sec: 0.90 + 28%|██████████████████████████████ | 12999/45917 [5:20:51<9:59:41, 1.09s/it][2025-04-23 22:43:03] (step=0013000) Train Loss: 6.3565, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▊ | 13024/45917 [5:21:19<10:22:10, 1.13s/it][2025-04-23 22:43:30] (step=0013025) Train Loss: 6.4312, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▊ | 13049/45917 [5:21:47<10:05:59, 1.11s/it][2025-04-23 22:43:58] (step=0013050) Train Loss: 6.3999, Train Steps/Sec: 0.90 + 28%|█████████████████████████████▉ | 13074/45917 [5:22:15<10:08:10, 1.11s/it][2025-04-23 22:44:26] (step=0013075) Train Loss: 6.4112, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▏ | 13099/45917 [5:22:43<9:58:44, 1.09s/it][2025-04-23 22:44:54] (step=0013100) Train Loss: 6.3772, Train Steps/Sec: 0.90 + 29%|██████████████████████████████ | 13124/45917 [5:23:11<10:20:20, 1.14s/it][2025-04-23 22:45:22] (step=0013125) Train Loss: 6.4676, Train Steps/Sec: 0.90 + 29%|██████████████████████████████ | 13149/45917 [5:23:38<10:07:25, 1.11s/it][2025-04-23 22:45:50] (step=0013150) Train Loss: 6.4166, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▍ | 13174/45917 [5:24:06<9:59:11, 1.10s/it][2025-04-23 22:46:17] (step=0013175) Train Loss: 6.3983, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▍ | 13199/45917 [5:24:34<9:56:07, 1.09s/it][2025-04-23 22:46:45] (step=0013200) Train Loss: 6.4243, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▏ | 13224/45917 [5:25:02<10:12:07, 1.12s/it][2025-04-23 22:47:13] (step=0013225) Train Loss: 6.4027, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▌ | 13249/45917 [5:25:29<9:59:30, 1.10s/it][2025-04-23 22:47:40] (step=0013250) Train Loss: 6.3922, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▋ | 13274/45917 [5:25:57<9:59:56, 1.10s/it][2025-04-23 22:48:08] (step=0013275) Train Loss: 6.3902, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▋ | 13299/45917 [5:26:25<9:54:58, 1.09s/it][2025-04-23 22:48:36] (step=0013300) Train Loss: 6.4106, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▍ | 13324/45917 [5:26:53<10:13:46, 1.13s/it][2025-04-23 22:49:04] (step=0013325) Train Loss: 6.4415, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▌ | 13349/45917 [5:27:20<10:04:17, 1.11s/it][2025-04-23 22:49:32] (step=0013350) Train Loss: 6.4182, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▌ | 13374/45917 [5:27:48<10:00:23, 1.11s/it][2025-04-23 22:49:59] (step=0013375) Train Loss: 6.4239, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▉ | 13399/45917 [5:28:16<9:58:30, 1.10s/it][2025-04-23 22:50:27] (step=0013400) Train Loss: 6.4259, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▋ | 13424/45917 [5:28:44<10:12:17, 1.13s/it][2025-04-23 22:50:55] (step=0013425) Train Loss: 6.4601, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▊ | 13449/45917 [5:29:11<10:03:12, 1.11s/it][2025-04-23 22:51:23] (step=0013450) Train Loss: 6.4166, Train Steps/Sec: 0.90 + 29%|███████████████████████████████ | 13474/45917 [5:29:39<9:50:36, 1.09s/it][2025-04-23 22:51:50] (step=0013475) Train Loss: 6.4598, Train Steps/Sec: 0.90 + 29%|███████████████████████████████▏ | 13499/45917 [5:30:07<9:53:50, 1.10s/it][2025-04-23 22:52:18] (step=0013500) Train Loss: 6.4389, Train Steps/Sec: 0.90 + 29%|██████████████████████████████▉ | 13524/45917 [5:30:35<10:06:55, 1.12s/it][2025-04-23 22:52:46] (step=0013525) Train Loss: 6.3907, Train Steps/Sec: 0.90 + 30%|███████████████████████████████▎ | 13549/45917 [5:31:02<9:57:49, 1.11s/it][2025-04-23 22:53:14] (step=0013550) Train Loss: 6.4806, Train Steps/Sec: 0.90 + 30%|███████████████████████████████ | 13574/45917 [5:31:30<10:01:55, 1.12s/it][2025-04-23 22:53:41] (step=0013575) Train Loss: 6.4617, Train Steps/Sec: 0.90 + 30%|███████████████████████████████▍ | 13599/45917 [5:31:58<9:53:43, 1.10s/it][2025-04-23 22:54:09] (step=0013600) Train Loss: 6.4138, Train Steps/Sec: 0.90 + 30%|███████████████████████████████▏ | 13624/45917 [5:32:26<10:08:34, 1.13s/it][2025-04-23 22:54:37] (step=0013625) Train Loss: 6.4470, Train Steps/Sec: 0.91 + 30%|███████████████████████████████▌ | 13649/45917 [5:32:53<9:53:08, 1.10s/it][2025-04-23 22:55:05] (step=0013650) Train Loss: 6.4197, Train Steps/Sec: 0.90 + 30%|███████████████████████████████▌ | 13674/45917 [5:33:21<9:59:20, 1.12s/it][2025-04-23 22:55:32] (step=0013675) Train Loss: 6.4809, Train Steps/Sec: 0.90 + 30%|███████████████████████████████▌ | 13699/45917 [5:33:49<9:50:13, 1.10s/it][2025-04-23 22:56:00] (step=0013700) Train Loss: 6.3521, Train Steps/Sec: 0.90 + 30%|███████████████████████████████▍ | 13724/45917 [5:34:17<10:06:59, 1.13s/it][2025-04-23 22:56:28] (step=0013725) Train Loss: 6.4107, Train Steps/Sec: 0.90 + 30%|███████████████████████████████▋ | 13749/45917 [5:34:44<9:53:27, 1.11s/it][2025-04-23 22:56:55] (step=0013750) Train Loss: 6.3956, Train Steps/Sec: 0.91 + 30%|███████████████████████████████▊ | 13774/45917 [5:35:12<9:51:11, 1.10s/it][2025-04-23 22:57:23] (step=0013775) Train Loss: 6.4454, Train Steps/Sec: 0.90 + 30%|███████████████████████████████▊ | 13799/45917 [5:35:40<9:45:56, 1.09s/it][2025-04-23 22:57:51] (step=0013800) Train Loss: 6.3623, Train Steps/Sec: 0.90 + 30%|███████████████████████████████▉ | 13824/45917 [5:36:07<9:59:50, 1.12s/it][2025-04-23 22:58:19] (step=0013825) Train Loss: 6.4187, Train Steps/Sec: 0.90 + 30%|███████████████████████████████▉ | 13849/45917 [5:36:35<9:57:31, 1.12s/it][2025-04-23 22:58:46] (step=0013850) Train Loss: 6.4162, Train Steps/Sec: 0.90 + 30%|████████████████████████████████ | 13874/45917 [5:37:03<9:51:26, 1.11s/it][2025-04-23 22:59:14] (step=0013875) Train Loss: 6.4023, Train Steps/Sec: 0.90 + 30%|████████████████████████████████ | 13899/45917 [5:37:31<9:41:16, 1.09s/it][2025-04-23 22:59:42] (step=0013900) Train Loss: 6.4363, Train Steps/Sec: 0.90 + 30%|███████████████████████████████▊ | 13924/45917 [5:37:58<10:01:49, 1.13s/it][2025-04-23 23:00:10] (step=0013925) Train Loss: 6.4498, Train Steps/Sec: 0.90 + 30%|████████████████████████████████▏ | 13949/45917 [5:38:26<9:55:56, 1.12s/it][2025-04-23 23:00:37] (step=0013950) Train Loss: 6.4581, Train Steps/Sec: 0.90 + 30%|████████████████████████████████▎ | 13974/45917 [5:38:54<9:44:32, 1.10s/it][2025-04-23 23:01:05] (step=0013975) Train Loss: 6.4093, Train Steps/Sec: 0.90 + 30%|████████████████████████████████▎ | 13999/45917 [5:39:22<9:39:57, 1.09s/it][2025-04-23 23:01:33] (step=0014000) Train Loss: 6.3570, Train Steps/Sec: 0.90 +[2025-04-23 23:01:33] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-23 23:01:33] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-23 23:01:33] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:59<00:00, 59.82s/it] +[2025-04-23 23:06:24] Finish Eval in 14000 steps...███████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.24s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 23:06:45] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0014000.pt +[2025-04-23 23:06:47] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0012000.pt + 31%|████████████████████████████████ | 14024/45917 [5:45:04<10:09:32, 1.15s/it][2025-04-23 23:07:15] (step=0014025) Train Loss: 6.4069, Train Steps/Sec: 0.07 + 31%|████████████████████████████████▏ | 14049/45917 [5:45:38<10:00:46, 1.13s/it][2025-04-23 23:07:49] (step=0014050) Train Loss: 6.4249, Train Steps/Sec: 0.74 + 31%|████████████████████████████████▍ | 14074/45917 [5:46:05<9:45:06, 1.10s/it][2025-04-23 23:08:23] (step=0014075) Train Loss: 6.4217, Train Steps/Sec: 0.74 + 31%|████████████████████████████████▌ | 14099/45917 [5:46:39<9:47:54, 1.11s/it][2025-04-23 23:08:51] (step=0014100) Train Loss: 6.3721, Train Steps/Sec: 0.90 + 31%|████████████████████████████████▌ | 14124/45917 [5:47:07<9:57:43, 1.13s/it][2025-04-23 23:09:18] (step=0014125) Train Loss: 6.4240, Train Steps/Sec: 0.90 + 31%|████████████████████████████████▋ | 14149/45917 [5:47:35<9:51:30, 1.12s/it][2025-04-23 23:09:46] (step=0014150) Train Loss: 6.4663, Train Steps/Sec: 0.90 + 31%|████████████████████████████████▋ | 14174/45917 [5:48:02<9:43:10, 1.10s/it][2025-04-23 23:10:14] (step=0014175) Train Loss: 6.4032, Train Steps/Sec: 0.90 + 31%|████████████████████████████████▍ | 14199/45917 [5:48:39<25:23:12, 2.88s/it][2025-04-23 23:10:50] (step=0014200) Train Loss: 6.4513, Train Steps/Sec: 0.69 + 31%|████████████████████████████████▊ | 14224/45917 [5:49:07<9:54:13, 1.12s/it][2025-04-23 23:11:18] (step=0014225) Train Loss: 6.4084, Train Steps/Sec: 0.91 + 31%|████████████████████████████████▌ | 14249/45917 [5:49:40<13:37:23, 1.55s/it][2025-04-23 23:11:51] (step=0014250) Train Loss: 6.4423, Train Steps/Sec: 0.74 + 31%|████████████████████████████████▉ | 14274/45917 [5:50:08<9:50:47, 1.12s/it][2025-04-23 23:12:19] (step=0014275) Train Loss: 6.4103, Train Steps/Sec: 0.90 + 31%|█████████████████████████████████ | 14299/45917 [5:50:42<9:43:47, 1.11s/it][2025-04-23 23:12:54] (step=0014300) Train Loss: 6.4483, Train Steps/Sec: 0.72 + 31%|████████████████████████████████▊ | 14324/45917 [5:51:17<11:48:08, 1.34s/it][2025-04-23 23:13:28] (step=0014325) Train Loss: 6.4438, Train Steps/Sec: 0.74 + 31%|█████████████████████████████████ | 14349/45917 [5:51:51<9:57:27, 1.14s/it][2025-04-23 23:14:02] (step=0014350) Train Loss: 6.4186, Train Steps/Sec: 0.74 + 31%|█████████████████████████████████▏ | 14374/45917 [5:52:18<9:37:36, 1.10s/it][2025-04-23 23:14:30] (step=0014375) Train Loss: 6.4442, Train Steps/Sec: 0.90 + 31%|█████████████████████████████████▏ | 14399/45917 [5:52:46<9:39:00, 1.10s/it][2025-04-23 23:14:58] (step=0014400) Train Loss: 6.4841, Train Steps/Sec: 0.90 + 31%|█████████████████████████████████▎ | 14424/45917 [5:53:14<9:57:50, 1.14s/it][2025-04-23 23:15:25] (step=0014425) Train Loss: 6.4703, Train Steps/Sec: 0.90 + 31%|█████████████████████████████████▎ | 14449/45917 [5:53:42<9:44:21, 1.11s/it][2025-04-23 23:15:53] (step=0014450) Train Loss: 6.3871, Train Steps/Sec: 0.90 + 32%|█████████████████████████████████▍ | 14474/45917 [5:54:10<9:35:34, 1.10s/it][2025-04-23 23:16:21] (step=0014475) Train Loss: 6.4210, Train Steps/Sec: 0.90 + 32%|█████████████████████████████████▍ | 14499/45917 [5:54:37<9:35:48, 1.10s/it][2025-04-23 23:16:49] (step=0014500) Train Loss: 6.4289, Train Steps/Sec: 0.90 + 32%|█████████████████████████████████▌ | 14524/45917 [5:55:05<9:51:15, 1.13s/it][2025-04-23 23:17:16] (step=0014525) Train Loss: 6.4364, Train Steps/Sec: 0.90 + 32%|█████████████████████████████████▌ | 14549/45917 [5:55:33<9:42:16, 1.11s/it][2025-04-23 23:17:44] (step=0014550) Train Loss: 6.3982, Train Steps/Sec: 0.90 + 32%|█████████████████████████████████▋ | 14574/45917 [5:56:01<9:41:18, 1.11s/it][2025-04-23 23:18:12] (step=0014575) Train Loss: 6.4160, Train Steps/Sec: 0.90 + 32%|█████████████████████████████████▋ | 14599/45917 [5:56:29<9:35:16, 1.10s/it][2025-04-23 23:18:40] (step=0014600) Train Loss: 6.4455, Train Steps/Sec: 0.89 + 32%|█████████████████████████████████▊ | 14624/45917 [5:56:56<9:49:52, 1.13s/it][2025-04-23 23:19:08] (step=0014625) Train Loss: 6.3876, Train Steps/Sec: 0.90 + 32%|█████████████████████████████████▊ | 14649/45917 [5:57:24<9:38:16, 1.11s/it][2025-04-23 23:19:35] (step=0014650) Train Loss: 6.4126, Train Steps/Sec: 0.91 + 32%|█████████████████████████████████▉ | 14674/45917 [5:57:52<9:37:20, 1.11s/it][2025-04-23 23:20:03] (step=0014675) Train Loss: 6.4358, Train Steps/Sec: 0.90 + 32%|█████████████████████████████████▉ | 14699/45917 [5:58:20<9:29:41, 1.09s/it][2025-04-23 23:20:31] (step=0014700) Train Loss: 6.4190, Train Steps/Sec: 0.89 + 32%|█████████████████████████████████▉ | 14724/45917 [5:58:47<9:43:51, 1.12s/it][2025-04-23 23:20:59] (step=0014725) Train Loss: 6.4597, Train Steps/Sec: 0.90 + 32%|██████████████████████████████████ | 14749/45917 [5:59:15<9:34:58, 1.11s/it][2025-04-23 23:21:26] (step=0014750) Train Loss: 6.4257, Train Steps/Sec: 0.90 + 32%|██████████████████████████████████ | 14774/45917 [5:59:43<9:32:37, 1.10s/it][2025-04-23 23:21:54] (step=0014775) Train Loss: 6.4225, Train Steps/Sec: 0.90 + 32%|██████████████████████████████████▏ | 14799/45917 [6:00:11<9:31:36, 1.10s/it][2025-04-23 23:22:22] (step=0014800) Train Loss: 6.4604, Train Steps/Sec: 0.89 + 32%|██████████████████████████████████▏ | 14824/45917 [6:00:39<9:46:45, 1.13s/it][2025-04-23 23:22:50] (step=0014825) Train Loss: 6.4084, Train Steps/Sec: 0.90 + 32%|██████████████████████████████████▎ | 14849/45917 [6:01:06<9:30:13, 1.10s/it][2025-04-23 23:23:18] (step=0014850) Train Loss: 6.4471, Train Steps/Sec: 0.90 + 32%|██████████████████████████████████▎ | 14874/45917 [6:01:34<9:26:28, 1.09s/it][2025-04-23 23:23:45] (step=0014875) Train Loss: 6.4220, Train Steps/Sec: 0.91 + 32%|██████████████████████████████████▍ | 14899/45917 [6:02:02<9:24:25, 1.09s/it][2025-04-23 23:24:13] (step=0014900) Train Loss: 6.4010, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████▍ | 14924/45917 [6:02:30<9:48:21, 1.14s/it][2025-04-23 23:24:41] (step=0014925) Train Loss: 6.4408, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████▌ | 14949/45917 [6:02:58<9:34:08, 1.11s/it][2025-04-23 23:25:09] (step=0014950) Train Loss: 6.4411, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████▌ | 14974/45917 [6:03:25<9:27:30, 1.10s/it][2025-04-23 23:25:36] (step=0014975) Train Loss: 6.3987, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████▋ | 14999/45917 [6:03:53<9:23:43, 1.09s/it][2025-04-23 23:26:04] (step=0015000) Train Loss: 6.4016, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████▋ | 15024/45917 [6:04:21<9:40:22, 1.13s/it][2025-04-23 23:26:32] (step=0015025) Train Loss: 6.4070, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████▋ | 15049/45917 [6:04:49<9:33:34, 1.11s/it][2025-04-23 23:27:00] (step=0015050) Train Loss: 6.4309, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████▊ | 15074/45917 [6:05:16<9:26:21, 1.10s/it][2025-04-23 23:27:27] (step=0015075) Train Loss: 6.4219, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████▊ | 15099/45917 [6:05:44<9:23:26, 1.10s/it][2025-04-23 23:27:55] (step=0015100) Train Loss: 6.4051, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████▉ | 15124/45917 [6:06:12<9:34:24, 1.12s/it][2025-04-23 23:28:23] (step=0015125) Train Loss: 6.4575, Train Steps/Sec: 0.91 + 33%|██████████████████████████████████▉ | 15149/45917 [6:06:39<9:30:49, 1.11s/it][2025-04-23 23:28:51] (step=0015150) Train Loss: 6.4106, Train Steps/Sec: 0.90 + 33%|███████████████████████████████████ | 15174/45917 [6:07:07<9:27:03, 1.11s/it][2025-04-23 23:29:18] (step=0015175) Train Loss: 6.4388, Train Steps/Sec: 0.90 + 33%|███████████████████████████████████ | 15199/45917 [6:07:35<9:24:18, 1.10s/it][2025-04-23 23:29:46] (step=0015200) Train Loss: 6.4233, Train Steps/Sec: 0.89 + 33%|███████████████████████████████████▏ | 15224/45917 [6:08:03<9:42:46, 1.14s/it][2025-04-23 23:30:14] (step=0015225) Train Loss: 6.4283, Train Steps/Sec: 0.90 + 33%|███████████████████████████████████▏ | 15249/45917 [6:08:31<9:28:15, 1.11s/it][2025-04-23 23:30:42] (step=0015250) Train Loss: 6.4184, Train Steps/Sec: 0.90 + 33%|███████████████████████████████████▎ | 15274/45917 [6:08:59<9:23:01, 1.10s/it][2025-04-23 23:31:10] (step=0015275) Train Loss: 6.4709, Train Steps/Sec: 0.90 + 33%|███████████████████████████████████▎ | 15299/45917 [6:09:26<9:16:56, 1.09s/it][2025-04-23 23:31:38] (step=0015300) Train Loss: 6.4129, Train Steps/Sec: 0.90 + 33%|███████████████████████████████████▍ | 15324/45917 [6:09:54<9:38:11, 1.13s/it][2025-04-23 23:32:05] (step=0015325) Train Loss: 6.4343, Train Steps/Sec: 0.90 + 33%|███████████████████████████████████▍ | 15349/45917 [6:10:22<9:30:27, 1.12s/it][2025-04-23 23:32:33] (step=0015350) Train Loss: 6.4404, Train Steps/Sec: 0.90 + 33%|███████████████████████████████████▍ | 15374/45917 [6:10:50<9:19:58, 1.10s/it][2025-04-23 23:33:01] (step=0015375) Train Loss: 6.3798, Train Steps/Sec: 0.91 + 34%|███████████████████████████████████▌ | 15399/45917 [6:11:17<9:19:12, 1.10s/it][2025-04-23 23:33:28] (step=0015400) Train Loss: 6.4515, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████▌ | 15424/45917 [6:11:45<9:37:13, 1.14s/it][2025-04-23 23:33:56] (step=0015425) Train Loss: 6.4351, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████▋ | 15449/45917 [6:12:13<9:23:39, 1.11s/it][2025-04-23 23:34:24] (step=0015450) Train Loss: 6.3832, Train Steps/Sec: 0.91 + 34%|███████████████████████████████████▋ | 15474/45917 [6:12:40<9:20:46, 1.11s/it][2025-04-23 23:34:52] (step=0015475) Train Loss: 6.4467, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████▊ | 15499/45917 [6:13:08<9:15:54, 1.10s/it][2025-04-23 23:35:19] (step=0015500) Train Loss: 6.3410, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████▊ | 15524/45917 [6:13:36<9:30:40, 1.13s/it][2025-04-23 23:35:47] (step=0015525) Train Loss: 6.4063, Train Steps/Sec: 0.91 + 34%|███████████████████████████████████▉ | 15549/45917 [6:14:03<9:22:53, 1.11s/it][2025-04-23 23:36:15] (step=0015550) Train Loss: 6.3885, Train Steps/Sec: 0.91 + 34%|███████████████████████████████████▉ | 15574/45917 [6:14:31<9:19:04, 1.11s/it][2025-04-23 23:36:42] (step=0015575) Train Loss: 6.4216, Train Steps/Sec: 0.90 + 34%|████████████████████████████████████ | 15599/45917 [6:14:59<9:10:36, 1.09s/it][2025-04-23 23:37:10] (step=0015600) Train Loss: 6.3994, Train Steps/Sec: 0.90 + 34%|████████████████████████████████████ | 15624/45917 [6:15:27<9:25:09, 1.12s/it][2025-04-23 23:37:38] (step=0015625) Train Loss: 6.3574, Train Steps/Sec: 0.90 + 34%|████████████████████████████████████▏ | 15649/45917 [6:15:54<9:14:48, 1.10s/it][2025-04-23 23:38:05] (step=0015650) Train Loss: 6.3586, Train Steps/Sec: 0.90 + 34%|████████████████████████████████████▏ | 15674/45917 [6:16:22<9:14:42, 1.10s/it][2025-04-23 23:38:33] (step=0015675) Train Loss: 6.4478, Train Steps/Sec: 0.90 + 34%|████████████████████████████████████▏ | 15699/45917 [6:16:50<9:12:49, 1.10s/it][2025-04-23 23:39:01] (step=0015700) Train Loss: 6.4096, Train Steps/Sec: 0.90 + 34%|████████████████████████████████████▎ | 15724/45917 [6:17:17<9:25:42, 1.12s/it][2025-04-23 23:39:29] (step=0015725) Train Loss: 6.3951, Train Steps/Sec: 0.90 + 34%|████████████████████████████████████▎ | 15749/45917 [6:17:45<9:17:45, 1.11s/it][2025-04-23 23:39:56] (step=0015750) Train Loss: 6.3606, Train Steps/Sec: 0.90 + 34%|████████████████████████████████████▍ | 15774/45917 [6:18:13<9:13:40, 1.10s/it][2025-04-23 23:40:24] (step=0015775) Train Loss: 6.4744, Train Steps/Sec: 0.90 + 34%|████████████████████████████████████▍ | 15799/45917 [6:18:41<9:08:16, 1.09s/it][2025-04-23 23:40:52] (step=0015800) Train Loss: 6.4394, Train Steps/Sec: 0.90 + 34%|████████████████████████████████████▌ | 15824/45917 [6:19:09<9:28:23, 1.13s/it][2025-04-23 23:41:20] (step=0015825) Train Loss: 6.4190, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████▌ | 15849/45917 [6:19:36<9:19:24, 1.12s/it][2025-04-23 23:41:48] (step=0015850) Train Loss: 6.4168, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████▋ | 15874/45917 [6:20:04<9:12:58, 1.10s/it][2025-04-23 23:42:15] (step=0015875) Train Loss: 6.4190, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████▋ | 15899/45917 [6:20:32<9:06:57, 1.09s/it][2025-04-23 23:42:43] (step=0015900) Train Loss: 6.3524, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████▊ | 15924/45917 [6:21:00<9:22:33, 1.13s/it][2025-04-23 23:43:11] (step=0015925) Train Loss: 6.3855, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████▊ | 15949/45917 [6:21:27<9:14:42, 1.11s/it][2025-04-23 23:43:38] (step=0015950) Train Loss: 6.4000, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████▉ | 15974/45917 [6:21:55<9:09:47, 1.10s/it][2025-04-23 23:44:06] (step=0015975) Train Loss: 6.3934, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████▉ | 15999/45917 [6:22:23<9:07:55, 1.10s/it][2025-04-23 23:44:34] (step=0016000) Train Loss: 6.4070, Train Steps/Sec: 0.89 +[2025-04-23 23:44:34] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-23 23:44:34] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-23 23:44:34] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 59.25s/it] +[2025-04-23 23:49:21] Finish Eval in 16000 steps...███████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 58.83s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 23:49:43] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0016000.pt +[2025-04-23 23:49:45] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0014000.pt + 35%|████████████████████████████████████▉ | 16024/45917 [6:28:02<9:29:39, 1.14s/it][2025-04-23 23:50:13] (step=0016025) Train Loss: 6.3828, Train Steps/Sec: 0.07 + 35%|█████████████████████████████████████ | 16049/45917 [6:28:29<9:14:59, 1.11s/it][2025-04-23 23:50:41] (step=0016050) Train Loss: 6.4208, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████ | 16074/45917 [6:28:57<9:05:17, 1.10s/it][2025-04-23 23:51:08] (step=0016075) Train Loss: 6.3930, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████▏ | 16099/45917 [6:29:25<9:04:48, 1.10s/it][2025-04-23 23:51:36] (step=0016100) Train Loss: 6.3870, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████▏ | 16124/45917 [6:29:53<9:20:55, 1.13s/it][2025-04-23 23:52:04] (step=0016125) Train Loss: 6.3689, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████▎ | 16149/45917 [6:30:20<9:15:52, 1.12s/it][2025-04-23 23:52:32] (step=0016150) Train Loss: 6.3821, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████▎ | 16174/45917 [6:30:48<9:06:57, 1.10s/it][2025-04-23 23:52:59] (step=0016175) Train Loss: 6.3769, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████▍ | 16199/45917 [6:31:16<9:05:22, 1.10s/it][2025-04-23 23:53:27] (step=0016200) Train Loss: 6.4052, Train Steps/Sec: 0.89 + 35%|█████████████████████████████████████▍ | 16224/45917 [6:31:44<9:15:11, 1.12s/it][2025-04-23 23:53:55] (step=0016225) Train Loss: 6.3742, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████▌ | 16249/45917 [6:32:12<9:10:20, 1.11s/it][2025-04-23 23:54:23] (step=0016250) Train Loss: 6.4141, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████▌ | 16274/45917 [6:32:39<9:02:44, 1.10s/it][2025-04-23 23:54:51] (step=0016275) Train Loss: 6.3802, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████▋ | 16299/45917 [6:33:07<9:02:34, 1.10s/it][2025-04-23 23:55:18] (step=0016300) Train Loss: 6.4105, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████▋ | 16324/45917 [6:33:35<9:12:11, 1.12s/it][2025-04-23 23:55:46] (step=0016325) Train Loss: 6.4409, Train Steps/Sec: 0.91 + 36%|█████████████████████████████████████▋ | 16349/45917 [6:34:03<9:03:54, 1.10s/it][2025-04-23 23:56:14] (step=0016350) Train Loss: 6.4324, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████▊ | 16374/45917 [6:34:30<9:08:37, 1.11s/it][2025-04-23 23:56:41] (step=0016375) Train Loss: 6.4185, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████▊ | 16399/45917 [6:34:58<9:08:12, 1.11s/it][2025-04-23 23:57:09] (step=0016400) Train Loss: 6.3940, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████▉ | 16424/45917 [6:35:26<9:19:29, 1.14s/it][2025-04-23 23:57:37] (step=0016425) Train Loss: 6.4390, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████▉ | 16449/45917 [6:35:54<9:05:08, 1.11s/it][2025-04-23 23:58:05] (step=0016450) Train Loss: 6.3901, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████ | 16474/45917 [6:36:22<9:00:24, 1.10s/it][2025-04-23 23:58:33] (step=0016475) Train Loss: 6.4294, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████ | 16499/45917 [6:36:49<8:56:53, 1.10s/it][2025-04-23 23:59:01] (step=0016500) Train Loss: 6.4328, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████▏ | 16524/45917 [6:37:17<9:11:03, 1.12s/it][2025-04-23 23:59:28] (step=0016525) Train Loss: 6.4093, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████▏ | 16549/45917 [6:37:45<9:03:55, 1.11s/it][2025-04-23 23:59:56] (step=0016550) Train Loss: 6.3813, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████▎ | 16574/45917 [6:38:12<8:57:59, 1.10s/it][2025-04-24 00:00:24] (step=0016575) Train Loss: 6.4063, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████▎ | 16599/45917 [6:38:40<8:55:13, 1.10s/it][2025-04-24 00:00:51] (step=0016600) Train Loss: 6.4759, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████▍ | 16624/45917 [6:39:08<9:04:28, 1.12s/it][2025-04-24 00:01:19] (step=0016625) Train Loss: 6.4346, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████▍ | 16649/45917 [6:39:36<8:59:01, 1.11s/it][2025-04-24 00:01:47] (step=0016650) Train Loss: 6.4285, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████▍ | 16674/45917 [6:40:03<8:57:45, 1.10s/it][2025-04-24 00:02:15] (step=0016675) Train Loss: 6.3975, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████▌ | 16699/45917 [6:40:31<8:53:20, 1.10s/it][2025-04-24 00:02:43] (step=0016700) Train Loss: 6.4164, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████▌ | 16724/45917 [6:40:59<9:08:35, 1.13s/it][2025-04-24 00:03:10] (step=0016725) Train Loss: 6.4689, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████▋ | 16749/45917 [6:41:27<8:59:52, 1.11s/it][2025-04-24 00:03:38] (step=0016750) Train Loss: 6.4341, Train Steps/Sec: 0.90 + 37%|██████████████████████████████████████▋ | 16774/45917 [6:41:54<8:57:15, 1.11s/it][2025-04-24 00:04:06] (step=0016775) Train Loss: 6.3717, Train Steps/Sec: 0.90 + 37%|██████████████████████████████████████▊ | 16799/45917 [6:42:22<8:52:54, 1.10s/it][2025-04-24 00:04:33] (step=0016800) Train Loss: 6.4116, Train Steps/Sec: 0.90 + 37%|██████████████████████████████████████▊ | 16824/45917 [6:42:50<9:04:28, 1.12s/it][2025-04-24 00:05:01] (step=0016825) Train Loss: 6.4062, Train Steps/Sec: 0.91 + 37%|██████████████████████████████████████▉ | 16849/45917 [6:43:17<8:55:55, 1.11s/it][2025-04-24 00:05:29] (step=0016850) Train Loss: 6.4367, Train Steps/Sec: 0.90 + 37%|██████████████████████████████████████▉ | 16874/45917 [6:43:45<8:55:28, 1.11s/it][2025-04-24 00:05:56] (step=0016875) Train Loss: 6.4220, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████ | 16899/45917 [6:44:13<8:57:06, 1.11s/it][2025-04-24 00:06:24] (step=0016900) Train Loss: 6.4539, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████ | 16924/45917 [6:44:41<9:03:07, 1.12s/it][2025-04-24 00:06:52] (step=0016925) Train Loss: 6.4206, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████▏ | 16949/45917 [6:45:08<9:01:00, 1.12s/it][2025-04-24 00:07:20] (step=0016950) Train Loss: 6.3569, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████▏ | 16974/45917 [6:45:36<8:51:22, 1.10s/it][2025-04-24 00:07:47] (step=0016975) Train Loss: 6.3507, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████▏ | 16999/45917 [6:46:04<8:43:18, 1.09s/it][2025-04-24 00:08:15] (step=0017000) Train Loss: 6.4457, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████▎ | 17024/45917 [6:46:32<9:03:51, 1.13s/it][2025-04-24 00:08:43] (step=0017025) Train Loss: 6.4271, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████▎ | 17049/45917 [6:46:59<8:51:10, 1.10s/it][2025-04-24 00:09:11] (step=0017050) Train Loss: 6.4200, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████▍ | 17074/45917 [6:47:27<8:56:48, 1.12s/it][2025-04-24 00:09:38] (step=0017075) Train Loss: 6.3941, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████▍ | 17099/45917 [6:47:55<8:46:11, 1.10s/it][2025-04-24 00:10:06] (step=0017100) Train Loss: 6.4366, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████▌ | 17124/45917 [6:48:23<9:00:48, 1.13s/it][2025-04-24 00:10:34] (step=0017125) Train Loss: 6.4022, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████▌ | 17149/45917 [6:48:51<8:51:40, 1.11s/it][2025-04-24 00:11:02] (step=0017150) Train Loss: 6.4097, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████▋ | 17174/45917 [6:49:18<8:53:39, 1.11s/it][2025-04-24 00:11:30] (step=0017175) Train Loss: 6.4022, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████▋ | 17199/45917 [6:49:46<8:43:18, 1.09s/it][2025-04-24 00:11:57] (step=0017200) Train Loss: 6.3557, Train Steps/Sec: 0.90 + 38%|███████████████████████████████████████▊ | 17224/45917 [6:50:14<8:59:43, 1.13s/it][2025-04-24 00:12:25] (step=0017225) Train Loss: 6.3862, Train Steps/Sec: 0.90 + 38%|███████████████████████████████████████▊ | 17249/45917 [6:50:42<8:49:08, 1.11s/it][2025-04-24 00:12:53] (step=0017250) Train Loss: 6.3817, Train Steps/Sec: 0.90 + 38%|███████████████████████████████████████▉ | 17274/45917 [6:51:10<8:45:55, 1.10s/it][2025-04-24 00:13:21] (step=0017275) Train Loss: 6.4018, Train Steps/Sec: 0.90 + 38%|███████████████████████████████████████▉ | 17299/45917 [6:51:37<8:47:41, 1.11s/it][2025-04-24 00:13:49] (step=0017300) Train Loss: 6.4412, Train Steps/Sec: 0.89 + 38%|███████████████████████████████████████▉ | 17324/45917 [6:52:05<8:53:59, 1.12s/it][2025-04-24 00:14:16] (step=0017325) Train Loss: 6.4065, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████ | 17349/45917 [6:52:33<8:48:02, 1.11s/it][2025-04-24 00:14:44] (step=0017350) Train Loss: 6.4061, Train Steps/Sec: 0.91 + 38%|████████████████████████████████████████ | 17374/45917 [6:53:01<8:47:06, 1.11s/it][2025-04-24 00:15:12] (step=0017375) Train Loss: 6.3862, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▏ | 17399/45917 [6:53:29<8:40:09, 1.09s/it][2025-04-24 00:15:40] (step=0017400) Train Loss: 6.4313, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▏ | 17424/45917 [6:53:56<8:53:57, 1.12s/it][2025-04-24 00:16:08] (step=0017425) Train Loss: 6.4130, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▎ | 17449/45917 [6:54:24<8:47:05, 1.11s/it][2025-04-24 00:16:35] (step=0017450) Train Loss: 6.3994, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▎ | 17474/45917 [6:54:52<8:45:08, 1.11s/it][2025-04-24 00:17:03] (step=0017475) Train Loss: 6.4020, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▍ | 17499/45917 [6:55:20<8:40:12, 1.10s/it][2025-04-24 00:17:31] (step=0017500) Train Loss: 6.4276, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▍ | 17524/45917 [6:55:48<8:55:53, 1.13s/it][2025-04-24 00:17:59] (step=0017525) Train Loss: 6.3970, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▌ | 17549/45917 [6:56:15<8:46:04, 1.11s/it][2025-04-24 00:18:26] (step=0017550) Train Loss: 6.3830, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▌ | 17574/45917 [6:56:43<8:47:13, 1.12s/it][2025-04-24 00:18:54] (step=0017575) Train Loss: 6.4433, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▋ | 17599/45917 [6:57:11<8:37:16, 1.10s/it][2025-04-24 00:19:22] (step=0017600) Train Loss: 6.4101, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▋ | 17624/45917 [6:57:39<8:51:37, 1.13s/it][2025-04-24 00:19:50] (step=0017625) Train Loss: 6.3575, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▋ | 17649/45917 [6:58:06<8:46:24, 1.12s/it][2025-04-24 00:20:18] (step=0017650) Train Loss: 6.4021, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████▊ | 17674/45917 [6:58:34<8:41:17, 1.11s/it][2025-04-24 00:20:45] (step=0017675) Train Loss: 6.3924, Train Steps/Sec: 0.90 + 39%|████████████████████████████████████████▊ | 17699/45917 [6:59:02<8:36:54, 1.10s/it][2025-04-24 00:21:13] (step=0017700) Train Loss: 6.4184, Train Steps/Sec: 0.90 + 39%|████████████████████████████████████████▉ | 17724/45917 [6:59:29<8:49:23, 1.13s/it][2025-04-24 00:21:41] (step=0017725) Train Loss: 6.3916, Train Steps/Sec: 0.90 + 39%|████████████████████████████████████████▉ | 17749/45917 [6:59:57<8:43:53, 1.12s/it][2025-04-24 00:22:08] (step=0017750) Train Loss: 6.3850, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████ | 17774/45917 [7:00:25<8:37:15, 1.10s/it][2025-04-24 00:22:36] (step=0017775) Train Loss: 6.4621, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████ | 17799/45917 [7:00:53<8:31:33, 1.09s/it][2025-04-24 00:23:04] (step=0017800) Train Loss: 6.4073, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████▏ | 17824/45917 [7:01:20<8:49:53, 1.13s/it][2025-04-24 00:23:32] (step=0017825) Train Loss: 6.4300, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████▏ | 17849/45917 [7:01:48<8:34:40, 1.10s/it][2025-04-24 00:23:59] (step=0017850) Train Loss: 6.4210, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████▎ | 17874/45917 [7:02:16<8:34:56, 1.10s/it][2025-04-24 00:24:27] (step=0017875) Train Loss: 6.4156, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████▎ | 17899/45917 [7:02:44<8:34:13, 1.10s/it][2025-04-24 00:24:55] (step=0017900) Train Loss: 6.3932, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████▍ | 17924/45917 [7:03:11<8:47:23, 1.13s/it][2025-04-24 00:25:23] (step=0017925) Train Loss: 6.4165, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████▍ | 17949/45917 [7:03:39<8:35:14, 1.11s/it][2025-04-24 00:25:50] (step=0017950) Train Loss: 6.3675, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████▍ | 17974/45917 [7:04:07<8:35:03, 1.11s/it][2025-04-24 00:26:18] (step=0017975) Train Loss: 6.4044, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████▌ | 17999/45917 [7:04:35<8:35:13, 1.11s/it][2025-04-24 00:26:46] (step=0018000) Train Loss: 6.4260, Train Steps/Sec: 0.90 +[2025-04-24 00:26:46] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 00:26:46] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 00:26:46] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 59.12s/it] +[2025-04-24 00:31:32] Finish Eval in 18000 steps...███████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.69s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 00:31:54] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0018000.pt +[2025-04-24 00:31:56] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0016000.pt + 39%|█████████████████████████████████████████▌ | 18024/45917 [7:10:13<8:49:37, 1.14s/it][2025-04-24 00:32:24] (step=0018025) Train Loss: 6.4167, Train Steps/Sec: 0.07 + 39%|█████████████████████████████████████████▋ | 18049/45917 [7:10:41<8:39:50, 1.12s/it][2025-04-24 00:32:52] (step=0018050) Train Loss: 6.3667, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████▋ | 18074/45917 [7:11:09<8:30:47, 1.10s/it][2025-04-24 00:33:20] (step=0018075) Train Loss: 6.4184, Train Steps/Sec: 0.89 + 39%|█████████████████████████████████████████▊ | 18099/45917 [7:11:36<8:29:14, 1.10s/it][2025-04-24 00:33:48] (step=0018100) Train Loss: 6.4380, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████▊ | 18124/45917 [7:12:04<8:44:03, 1.13s/it][2025-04-24 00:34:15] (step=0018125) Train Loss: 6.4423, Train Steps/Sec: 0.90 + 40%|█████████████████████████████████████████▉ | 18149/45917 [7:12:32<8:38:33, 1.12s/it][2025-04-24 00:34:43] (step=0018150) Train Loss: 6.3847, Train Steps/Sec: 0.90 + 40%|█████████████████████████████████████████▉ | 18174/45917 [7:13:00<8:30:30, 1.10s/it][2025-04-24 00:35:11] (step=0018175) Train Loss: 6.3852, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████ | 18199/45917 [7:13:27<8:23:36, 1.09s/it][2025-04-24 00:35:39] (step=0018200) Train Loss: 6.4160, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████ | 18224/45917 [7:13:55<8:37:15, 1.12s/it][2025-04-24 00:36:06] (step=0018225) Train Loss: 6.4132, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████▏ | 18249/45917 [7:14:23<8:33:58, 1.11s/it][2025-04-24 00:36:34] (step=0018250) Train Loss: 6.4058, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████▏ | 18274/45917 [7:14:50<8:24:29, 1.10s/it][2025-04-24 00:37:02] (step=0018275) Train Loss: 6.4239, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████▏ | 18299/45917 [7:15:18<8:24:44, 1.10s/it][2025-04-24 00:37:30] (step=0018300) Train Loss: 6.3581, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████▎ | 18324/45917 [7:15:46<8:36:35, 1.12s/it][2025-04-24 00:37:57] (step=0018325) Train Loss: 6.3927, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████▎ | 18349/45917 [7:16:14<8:36:16, 1.12s/it][2025-04-24 00:38:25] (step=0018350) Train Loss: 6.4026, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████▍ | 18374/45917 [7:16:42<8:25:36, 1.10s/it][2025-04-24 00:38:53] (step=0018375) Train Loss: 6.3587, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████▍ | 18399/45917 [7:17:09<8:21:04, 1.09s/it][2025-04-24 00:39:21] (step=0018400) Train Loss: 6.4135, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████▌ | 18424/45917 [7:17:37<8:35:47, 1.13s/it][2025-04-24 00:39:48] (step=0018425) Train Loss: 6.4094, Train Steps/Sec: 0.91 + 40%|██████████████████████████████████████████▌ | 18449/45917 [7:18:05<8:30:56, 1.12s/it][2025-04-24 00:40:16] (step=0018450) Train Loss: 6.4031, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████▋ | 18474/45917 [7:18:33<8:21:50, 1.10s/it][2025-04-24 00:40:44] (step=0018475) Train Loss: 6.4212, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████▋ | 18499/45917 [7:19:00<8:21:02, 1.10s/it][2025-04-24 00:41:12] (step=0018500) Train Loss: 6.3912, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████▊ | 18524/45917 [7:19:28<8:34:32, 1.13s/it][2025-04-24 00:41:39] (step=0018525) Train Loss: 6.3958, Train Steps/Sec: 0.91 + 40%|██████████████████████████████████████████▊ | 18549/45917 [7:19:56<8:20:50, 1.10s/it][2025-04-24 00:42:07] (step=0018550) Train Loss: 6.3843, Train Steps/Sec: 0.91 + 40%|██████████████████████████████████████████▉ | 18574/45917 [7:20:23<8:25:04, 1.11s/it][2025-04-24 00:42:34] (step=0018575) Train Loss: 6.4213, Train Steps/Sec: 0.90 + 41%|██████████████████████████████████████████▉ | 18599/45917 [7:20:51<8:17:50, 1.09s/it][2025-04-24 00:43:02] (step=0018600) Train Loss: 6.4353, Train Steps/Sec: 0.90 + 41%|██████████████████████████████████████████▉ | 18624/45917 [7:21:19<8:33:01, 1.13s/it][2025-04-24 00:43:30] (step=0018625) Train Loss: 6.4270, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████ | 18649/45917 [7:21:46<8:22:02, 1.10s/it][2025-04-24 00:43:58] (step=0018650) Train Loss: 6.4266, Train Steps/Sec: 0.91 + 41%|███████████████████████████████████████████ | 18674/45917 [7:22:14<8:19:51, 1.10s/it][2025-04-24 00:44:25] (step=0018675) Train Loss: 6.3807, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▏ | 18699/45917 [7:22:42<8:20:56, 1.10s/it][2025-04-24 00:44:53] (step=0018700) Train Loss: 6.3998, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▏ | 18724/45917 [7:23:10<8:34:58, 1.14s/it][2025-04-24 00:45:21] (step=0018725) Train Loss: 6.3975, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▎ | 18749/45917 [7:23:37<8:22:22, 1.11s/it][2025-04-24 00:45:49] (step=0018750) Train Loss: 6.4121, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▎ | 18774/45917 [7:24:05<8:23:08, 1.11s/it][2025-04-24 00:46:16] (step=0018775) Train Loss: 6.4371, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▍ | 18799/45917 [7:24:33<8:12:53, 1.09s/it][2025-04-24 00:46:44] (step=0018800) Train Loss: 6.3748, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▍ | 18824/45917 [7:25:01<8:30:43, 1.13s/it][2025-04-24 00:47:12] (step=0018825) Train Loss: 6.3856, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▌ | 18849/45917 [7:25:28<8:17:00, 1.10s/it][2025-04-24 00:47:39] (step=0018850) Train Loss: 6.4589, Train Steps/Sec: 0.91 + 41%|███████████████████████████████████████████▌ | 18874/45917 [7:25:56<8:15:44, 1.10s/it][2025-04-24 00:48:07] (step=0018875) Train Loss: 6.4012, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▋ | 18899/45917 [7:26:24<8:12:59, 1.09s/it][2025-04-24 00:48:35] (step=0018900) Train Loss: 6.3544, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▋ | 18924/45917 [7:26:51<8:21:15, 1.11s/it][2025-04-24 00:49:03] (step=0018925) Train Loss: 6.3425, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▋ | 18949/45917 [7:27:19<8:20:13, 1.11s/it][2025-04-24 00:49:31] (step=0018950) Train Loss: 6.4818, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▊ | 18974/45917 [7:27:47<8:12:07, 1.10s/it][2025-04-24 00:49:58] (step=0018975) Train Loss: 6.3935, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▊ | 18999/45917 [7:28:15<8:15:32, 1.10s/it][2025-04-24 00:50:26] (step=0019000) Train Loss: 6.3795, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▉ | 19024/45917 [7:28:43<8:24:54, 1.13s/it][2025-04-24 00:50:54] (step=0019025) Train Loss: 6.3722, Train Steps/Sec: 0.90 + 41%|███████████████████████████████████████████▉ | 19049/45917 [7:29:10<8:17:25, 1.11s/it][2025-04-24 00:51:22] (step=0019050) Train Loss: 6.4750, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████ | 19074/45917 [7:29:38<8:14:47, 1.11s/it][2025-04-24 00:51:49] (step=0019075) Train Loss: 6.3761, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████ | 19099/45917 [7:30:06<8:08:46, 1.09s/it][2025-04-24 00:52:17] (step=0019100) Train Loss: 6.4355, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▏ | 19124/45917 [7:30:34<8:22:59, 1.13s/it][2025-04-24 00:52:45] (step=0019125) Train Loss: 6.3744, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▏ | 19149/45917 [7:31:01<8:15:08, 1.11s/it][2025-04-24 00:53:13] (step=0019150) Train Loss: 6.4277, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▎ | 19174/45917 [7:31:29<8:07:36, 1.09s/it][2025-04-24 00:53:40] (step=0019175) Train Loss: 6.4085, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▎ | 19199/45917 [7:31:57<8:07:14, 1.09s/it][2025-04-24 00:54:08] (step=0019200) Train Loss: 6.4209, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▍ | 19224/45917 [7:32:24<8:19:05, 1.12s/it][2025-04-24 00:54:36] (step=0019225) Train Loss: 6.3851, Train Steps/Sec: 0.91 + 42%|████████████████████████████████████████████▍ | 19249/45917 [7:32:52<8:15:32, 1.11s/it][2025-04-24 00:55:03] (step=0019250) Train Loss: 6.3999, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▍ | 19274/45917 [7:33:20<8:10:10, 1.10s/it][2025-04-24 00:55:31] (step=0019275) Train Loss: 6.3657, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▌ | 19299/45917 [7:33:48<8:05:59, 1.10s/it][2025-04-24 00:55:59] (step=0019300) Train Loss: 6.3978, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▌ | 19324/45917 [7:34:15<8:23:37, 1.14s/it][2025-04-24 00:56:27] (step=0019325) Train Loss: 6.3839, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▋ | 19349/45917 [7:34:43<8:13:16, 1.11s/it][2025-04-24 00:56:54] (step=0019350) Train Loss: 6.3805, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▋ | 19374/45917 [7:35:11<8:06:48, 1.10s/it][2025-04-24 00:57:22] (step=0019375) Train Loss: 6.3525, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▊ | 19399/45917 [7:35:39<8:08:08, 1.10s/it][2025-04-24 00:57:50] (step=0019400) Train Loss: 6.4110, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▊ | 19424/45917 [7:36:07<8:20:54, 1.13s/it][2025-04-24 00:58:18] (step=0019425) Train Loss: 6.4303, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▉ | 19449/45917 [7:36:34<8:08:46, 1.11s/it][2025-04-24 00:58:46] (step=0019450) Train Loss: 6.3722, Train Steps/Sec: 0.90 + 42%|████████████████████████████████████████████▉ | 19474/45917 [7:37:02<8:09:48, 1.11s/it][2025-04-24 00:59:13] (step=0019475) Train Loss: 6.4153, Train Steps/Sec: 0.90 + 42%|█████████████████████████████████████████████ | 19499/45917 [7:37:36<8:48:56, 1.20s/it][2025-04-24 00:59:47] (step=0019500) Train Loss: 6.3991, Train Steps/Sec: 0.73 + 43%|█████████████████████████████████████████████ | 19524/45917 [7:38:10<8:49:45, 1.20s/it][2025-04-24 01:00:21] (step=0019525) Train Loss: 6.3699, Train Steps/Sec: 0.74 + 43%|█████████████████████████████████████████████▏ | 19549/45917 [7:38:38<8:08:12, 1.11s/it][2025-04-24 01:00:49] (step=0019550) Train Loss: 6.4588, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▏ | 19574/45917 [7:39:05<8:06:43, 1.11s/it][2025-04-24 01:01:17] (step=0019575) Train Loss: 6.4022, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▏ | 19599/45917 [7:39:33<8:02:51, 1.10s/it][2025-04-24 01:01:45] (step=0019600) Train Loss: 6.3316, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▎ | 19624/45917 [7:40:01<8:14:07, 1.13s/it][2025-04-24 01:02:12] (step=0019625) Train Loss: 6.3807, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▎ | 19649/45917 [7:40:29<8:01:14, 1.10s/it][2025-04-24 01:02:40] (step=0019650) Train Loss: 6.3412, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▍ | 19674/45917 [7:40:56<8:06:09, 1.11s/it][2025-04-24 01:03:08] (step=0019675) Train Loss: 6.4118, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▍ | 19699/45917 [7:41:24<7:58:47, 1.10s/it][2025-04-24 01:03:35] (step=0019700) Train Loss: 6.3577, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▌ | 19724/45917 [7:41:58<8:10:23, 1.12s/it][2025-04-24 01:04:09] (step=0019725) Train Loss: 6.4048, Train Steps/Sec: 0.74 + 43%|█████████████████████████████████████████████▌ | 19749/45917 [7:42:26<8:03:27, 1.11s/it][2025-04-24 01:04:37] (step=0019750) Train Loss: 6.4139, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▋ | 19774/45917 [7:42:53<7:57:46, 1.10s/it][2025-04-24 01:05:05] (step=0019775) Train Loss: 6.3916, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▋ | 19799/45917 [7:43:27<7:59:56, 1.10s/it][2025-04-24 01:05:39] (step=0019800) Train Loss: 6.4321, Train Steps/Sec: 0.73 + 43%|█████████████████████████████████████████████▊ | 19824/45917 [7:44:01<8:09:12, 1.12s/it][2025-04-24 01:06:12] (step=0019825) Train Loss: 6.4305, Train Steps/Sec: 0.74 + 43%|█████████████████████████████████████████████▊ | 19849/45917 [7:44:29<8:01:13, 1.11s/it][2025-04-24 01:06:40] (step=0019850) Train Loss: 6.3816, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▉ | 19874/45917 [7:44:57<8:00:34, 1.11s/it][2025-04-24 01:07:08] (step=0019875) Train Loss: 6.4165, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▉ | 19899/45917 [7:45:24<7:55:44, 1.10s/it][2025-04-24 01:07:36] (step=0019900) Train Loss: 6.4549, Train Steps/Sec: 0.90 + 43%|█████████████████████████████████████████████▉ | 19924/45917 [7:45:52<8:14:06, 1.14s/it][2025-04-24 01:08:03] (step=0019925) Train Loss: 6.3722, Train Steps/Sec: 0.90 + 43%|██████████████████████████████████████████████ | 19949/45917 [7:46:20<7:58:35, 1.11s/it][2025-04-24 01:08:31] (step=0019950) Train Loss: 6.4058, Train Steps/Sec: 0.90 + 44%|██████████████████████████████████████████████ | 19974/45917 [7:46:48<7:59:12, 1.11s/it][2025-04-24 01:08:59] (step=0019975) Train Loss: 6.4474, Train Steps/Sec: 0.90 + 44%|██████████████████████████████████████████████▏ | 19999/45917 [7:47:15<7:52:14, 1.09s/it][2025-04-24 01:09:27] (step=0020000) Train Loss: 6.4273, Train Steps/Sec: 0.90 +[2025-04-24 01:09:27] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 01:09:27] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 01:09:27] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 59.09s/it] +[2025-04-24 01:14:13] Finish Eval in 20000 steps...███████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.67s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 01:14:35] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0020000.pt +[2025-04-24 01:14:37] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0018000.pt + 44%|██████████████████████████████████████████████▏ | 20024/45917 [7:52:54<8:15:26, 1.15s/it][2025-04-24 01:15:05] (step=0020025) Train Loss: 6.3567, Train Steps/Sec: 0.07 + 44%|█████████████████████████████████████████████▊ | 20049/45917 [7:53:29<23:21:30, 3.25s/it][2025-04-24 01:15:41] (step=0020050) Train Loss: 6.4120, Train Steps/Sec: 0.71 + 44%|██████████████████████████████████████████████▎ | 20074/45917 [7:53:57<7:53:01, 1.10s/it][2025-04-24 01:16:08] (step=0020075) Train Loss: 6.4298, Train Steps/Sec: 0.91 + 44%|██████████████████████████████████████████████▍ | 20099/45917 [7:54:25<7:53:16, 1.10s/it][2025-04-24 01:16:36] (step=0020100) Train Loss: 6.3975, Train Steps/Sec: 0.90 + 44%|██████████████████████████████████████████████▍ | 20124/45917 [7:54:53<8:03:53, 1.13s/it][2025-04-24 01:17:04] (step=0020125) Train Loss: 6.3835, Train Steps/Sec: 0.90 + 44%|██████████████████████████████████████████████▌ | 20149/45917 [7:55:27<8:10:40, 1.14s/it][2025-04-24 01:17:38] (step=0020150) Train Loss: 6.3907, Train Steps/Sec: 0.73 + 44%|██████████████████████████████████████████████▌ | 20174/45917 [7:55:55<7:54:40, 1.11s/it][2025-04-24 01:18:06] (step=0020175) Train Loss: 6.3981, Train Steps/Sec: 0.90 + 44%|██████████████████████████████████████████████▋ | 20199/45917 [7:56:22<7:49:50, 1.10s/it][2025-04-24 01:18:34] (step=0020200) Train Loss: 6.3981, Train Steps/Sec: 0.90 + 44%|██████████████████████████████████████████████▋ | 20224/45917 [7:56:50<8:05:19, 1.13s/it][2025-04-24 01:19:01] (step=0020225) Train Loss: 6.3650, Train Steps/Sec: 0.91 + 44%|██████████████████████████████████████████████▋ | 20249/45917 [7:57:18<7:58:01, 1.12s/it][2025-04-24 01:19:29] (step=0020250) Train Loss: 6.3776, Train Steps/Sec: 0.90 + 44%|██████████████████████████████████████████████▊ | 20274/45917 [7:57:45<7:50:51, 1.10s/it][2025-04-24 01:19:57] (step=0020275) Train Loss: 6.3552, Train Steps/Sec: 0.90 + 44%|██████████████████████████████████████████████▊ | 20299/45917 [7:58:13<7:47:06, 1.09s/it][2025-04-24 01:20:24] (step=0020300) Train Loss: 6.3908, Train Steps/Sec: 0.90 + 44%|██████████████████████████████████████████████▉ | 20324/45917 [7:58:41<8:00:04, 1.13s/it][2025-04-24 01:20:52] (step=0020325) Train Loss: 6.3577, Train Steps/Sec: 0.90 + 44%|██████████████████████████████████████████████▉ | 20349/45917 [7:59:08<7:47:40, 1.10s/it][2025-04-24 01:21:20] (step=0020350) Train Loss: 6.4373, Train Steps/Sec: 0.91 + 44%|███████████████████████████████████████████████ | 20374/45917 [7:59:36<7:51:25, 1.11s/it][2025-04-24 01:21:48] (step=0020375) Train Loss: 6.4245, Train Steps/Sec: 0.90 + 44%|███████████████████████████████████████████████ | 20399/45917 [8:00:04<7:49:31, 1.10s/it][2025-04-24 01:22:15] (step=0020400) Train Loss: 6.4324, Train Steps/Sec: 0.89 + 44%|███████████████████████████████████████████████▏ | 20424/45917 [8:00:32<7:59:11, 1.13s/it][2025-04-24 01:22:43] (step=0020425) Train Loss: 6.3776, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▏ | 20449/45917 [8:01:00<7:53:53, 1.12s/it][2025-04-24 01:23:11] (step=0020450) Train Loss: 6.4128, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▎ | 20474/45917 [8:01:28<7:48:15, 1.10s/it][2025-04-24 01:23:39] (step=0020475) Train Loss: 6.3933, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▎ | 20499/45917 [8:01:55<7:48:24, 1.11s/it][2025-04-24 01:24:07] (step=0020500) Train Loss: 6.3843, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▍ | 20524/45917 [8:02:23<8:00:48, 1.14s/it][2025-04-24 01:24:35] (step=0020525) Train Loss: 6.3645, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▍ | 20549/45917 [8:02:51<7:49:10, 1.11s/it][2025-04-24 01:25:02] (step=0020550) Train Loss: 6.3815, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▍ | 20574/45917 [8:03:19<7:47:21, 1.11s/it][2025-04-24 01:25:30] (step=0020575) Train Loss: 6.3970, Train Steps/Sec: 0.89 + 45%|███████████████████████████████████████████████▌ | 20599/45917 [8:03:47<7:42:05, 1.10s/it][2025-04-24 01:25:58] (step=0020600) Train Loss: 6.3834, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▌ | 20624/45917 [8:04:15<7:55:12, 1.13s/it][2025-04-24 01:26:26] (step=0020625) Train Loss: 6.4186, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▋ | 20649/45917 [8:04:42<7:44:30, 1.10s/it][2025-04-24 01:26:54] (step=0020650) Train Loss: 6.4085, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▋ | 20674/45917 [8:05:10<7:43:34, 1.10s/it][2025-04-24 01:27:21] (step=0020675) Train Loss: 6.4039, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▊ | 20699/45917 [8:05:38<7:42:37, 1.10s/it][2025-04-24 01:27:49] (step=0020700) Train Loss: 6.3976, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▊ | 20724/45917 [8:06:06<7:55:16, 1.13s/it][2025-04-24 01:28:17] (step=0020725) Train Loss: 6.4292, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▉ | 20749/45917 [8:06:33<7:46:25, 1.11s/it][2025-04-24 01:28:45] (step=0020750) Train Loss: 6.4006, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████▉ | 20774/45917 [8:07:01<7:43:15, 1.11s/it][2025-04-24 01:29:12] (step=0020775) Train Loss: 6.4246, Train Steps/Sec: 0.90 + 45%|████████████████████████████████████████████████ | 20799/45917 [8:07:29<7:39:40, 1.10s/it][2025-04-24 01:29:40] (step=0020800) Train Loss: 6.4286, Train Steps/Sec: 0.90 + 45%|████████████████████████████████████████████████ | 20824/45917 [8:07:57<7:51:54, 1.13s/it][2025-04-24 01:30:08] (step=0020825) Train Loss: 6.3898, Train Steps/Sec: 0.90 + 45%|████████████████████████████████████████████████▏ | 20849/45917 [8:08:24<7:45:43, 1.11s/it][2025-04-24 01:30:35] (step=0020850) Train Loss: 6.3795, Train Steps/Sec: 0.90 + 45%|████████████████████████████████████████████████▏ | 20874/45917 [8:08:52<7:39:36, 1.10s/it][2025-04-24 01:31:03] (step=0020875) Train Loss: 6.3851, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▏ | 20899/45917 [8:09:20<7:36:52, 1.10s/it][2025-04-24 01:31:31] (step=0020900) Train Loss: 6.4090, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▎ | 20924/45917 [8:09:47<7:48:23, 1.12s/it][2025-04-24 01:31:59] (step=0020925) Train Loss: 6.4019, Train Steps/Sec: 0.91 + 46%|████████████████████████████████████████████████▎ | 20949/45917 [8:10:15<7:42:32, 1.11s/it][2025-04-24 01:32:26] (step=0020950) Train Loss: 6.4080, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▍ | 20974/45917 [8:10:43<7:38:48, 1.10s/it][2025-04-24 01:32:54] (step=0020975) Train Loss: 6.4028, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▍ | 20999/45917 [8:11:11<7:32:55, 1.09s/it][2025-04-24 01:33:22] (step=0021000) Train Loss: 6.3893, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▌ | 21024/45917 [8:11:38<7:50:34, 1.13s/it][2025-04-24 01:33:50] (step=0021025) Train Loss: 6.4232, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▌ | 21049/45917 [8:12:06<7:39:03, 1.11s/it][2025-04-24 01:34:17] (step=0021050) Train Loss: 6.4303, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▋ | 21074/45917 [8:12:34<7:37:30, 1.10s/it][2025-04-24 01:34:45] (step=0021075) Train Loss: 6.3995, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▋ | 21099/45917 [8:13:02<7:33:45, 1.10s/it][2025-04-24 01:35:13] (step=0021100) Train Loss: 6.3980, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▊ | 21124/45917 [8:13:29<7:46:59, 1.13s/it][2025-04-24 01:35:40] (step=0021125) Train Loss: 6.3966, Train Steps/Sec: 0.91 + 46%|████████████████████████████████████████████████▊ | 21149/45917 [8:13:57<7:37:38, 1.11s/it][2025-04-24 01:36:08] (step=0021150) Train Loss: 6.3648, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▉ | 21174/45917 [8:14:25<7:32:33, 1.10s/it][2025-04-24 01:36:36] (step=0021175) Train Loss: 6.3943, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▉ | 21199/45917 [8:14:52<7:31:18, 1.10s/it][2025-04-24 01:37:04] (step=0021200) Train Loss: 6.3855, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████▉ | 21224/45917 [8:15:20<7:42:17, 1.12s/it][2025-04-24 01:37:31] (step=0021225) Train Loss: 6.4105, Train Steps/Sec: 0.90 + 46%|█████████████████████████████████████████████████ | 21249/45917 [8:15:48<7:37:20, 1.11s/it][2025-04-24 01:37:59] (step=0021250) Train Loss: 6.4277, Train Steps/Sec: 0.90 + 46%|█████████████████████████████████████████████████ | 21274/45917 [8:16:16<7:31:35, 1.10s/it][2025-04-24 01:38:27] (step=0021275) Train Loss: 6.3909, Train Steps/Sec: 0.90 + 46%|█████████████████████████████████████████████████▏ | 21299/45917 [8:16:43<7:33:53, 1.11s/it][2025-04-24 01:38:55] (step=0021300) Train Loss: 6.3556, Train Steps/Sec: 0.90 + 46%|█████████████████████████████████████████████████▏ | 21324/45917 [8:17:11<7:43:41, 1.13s/it][2025-04-24 01:39:22] (step=0021325) Train Loss: 6.4024, Train Steps/Sec: 0.90 + 46%|█████████████████████████████████████████████████▎ | 21349/45917 [8:17:39<7:35:18, 1.11s/it][2025-04-24 01:39:50] (step=0021350) Train Loss: 6.3717, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████▎ | 21374/45917 [8:18:07<7:27:15, 1.09s/it][2025-04-24 01:40:18] (step=0021375) Train Loss: 6.3893, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████▍ | 21399/45917 [8:18:34<7:31:57, 1.11s/it][2025-04-24 01:40:46] (step=0021400) Train Loss: 6.3934, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████▍ | 21424/45917 [8:19:02<7:47:23, 1.14s/it][2025-04-24 01:41:13] (step=0021425) Train Loss: 6.3981, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████▌ | 21449/45917 [8:19:30<7:37:54, 1.12s/it][2025-04-24 01:41:41] (step=0021450) Train Loss: 6.3980, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████▌ | 21474/45917 [8:19:58<7:29:29, 1.10s/it][2025-04-24 01:42:09] (step=0021475) Train Loss: 6.3816, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████▋ | 21499/45917 [8:20:25<7:26:04, 1.10s/it][2025-04-24 01:42:37] (step=0021500) Train Loss: 6.3624, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████▋ | 21524/45917 [8:20:53<7:38:39, 1.13s/it][2025-04-24 01:43:04] (step=0021525) Train Loss: 6.4366, Train Steps/Sec: 0.91 + 47%|█████████████████████████████████████████████████▋ | 21549/45917 [8:21:21<7:35:19, 1.12s/it][2025-04-24 01:43:32] (step=0021550) Train Loss: 6.4044, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████▊ | 21574/45917 [8:21:49<7:27:19, 1.10s/it][2025-04-24 01:44:00] (step=0021575) Train Loss: 6.3640, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████▊ | 21599/45917 [8:22:16<7:25:39, 1.10s/it][2025-04-24 01:44:28] (step=0021600) Train Loss: 6.3762, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████▉ | 21624/45917 [8:22:44<7:38:06, 1.13s/it][2025-04-24 01:44:55] (step=0021625) Train Loss: 6.4235, Train Steps/Sec: 0.91 + 47%|█████████████████████████████████████████████████▉ | 21649/45917 [8:23:12<7:31:08, 1.12s/it][2025-04-24 01:45:23] (step=0021650) Train Loss: 6.3924, Train Steps/Sec: 0.90 + 47%|██████████████████████████████████████████████████ | 21674/45917 [8:23:40<7:22:22, 1.09s/it][2025-04-24 01:45:51] (step=0021675) Train Loss: 6.4150, Train Steps/Sec: 0.90 + 47%|██████████████████████████████████████████████████ | 21699/45917 [8:24:07<7:22:55, 1.10s/it][2025-04-24 01:46:19] (step=0021700) Train Loss: 6.3721, Train Steps/Sec: 0.90 + 47%|██████████████████████████████████████████████████▏ | 21724/45917 [8:24:35<7:33:42, 1.13s/it][2025-04-24 01:46:46] (step=0021725) Train Loss: 6.3579, Train Steps/Sec: 0.90 + 47%|██████████████████████████████████████████████████▏ | 21749/45917 [8:25:03<7:26:36, 1.11s/it][2025-04-24 01:47:14] (step=0021750) Train Loss: 6.4040, Train Steps/Sec: 0.90 + 47%|██████████████████████████████████████████████████▎ | 21774/45917 [8:25:31<7:21:44, 1.10s/it][2025-04-24 01:47:42] (step=0021775) Train Loss: 6.4199, Train Steps/Sec: 0.91 + 47%|██████████████████████████████████████████████████▎ | 21799/45917 [8:25:58<7:20:23, 1.10s/it][2025-04-24 01:48:10] (step=0021800) Train Loss: 6.4517, Train Steps/Sec: 0.90 + 48%|██████████████████████████████████████████████████▍ | 21824/45917 [8:26:26<7:31:17, 1.12s/it][2025-04-24 01:48:37] (step=0021825) Train Loss: 6.3705, Train Steps/Sec: 0.91 + 48%|██████████████████████████████████████████████████▍ | 21849/45917 [8:26:54<7:25:28, 1.11s/it][2025-04-24 01:49:05] (step=0021850) Train Loss: 6.3977, Train Steps/Sec: 0.90 + 48%|██████████████████████████████████████████████████▍ | 21874/45917 [8:27:21<7:17:40, 1.09s/it][2025-04-24 01:49:32] (step=0021875) Train Loss: 6.3854, Train Steps/Sec: 0.91 + 48%|██████████████████████████████████████████████████▌ | 21899/45917 [8:27:49<7:19:29, 1.10s/it][2025-04-24 01:50:00] (step=0021900) Train Loss: 6.3930, Train Steps/Sec: 0.90 + 48%|██████████████████████████████████████████████████▌ | 21924/45917 [8:28:17<7:31:03, 1.13s/it][2025-04-24 01:50:28] (step=0021925) Train Loss: 6.4611, Train Steps/Sec: 0.90 + 48%|██████████████████████████████████████████████████▋ | 21949/45917 [8:28:44<7:24:37, 1.11s/it][2025-04-24 01:50:56] (step=0021950) Train Loss: 6.3890, Train Steps/Sec: 0.91 + 48%|██████████████████████████████████████████████████▋ | 21974/45917 [8:29:12<7:16:18, 1.09s/it][2025-04-24 01:51:23] (step=0021975) Train Loss: 6.3621, Train Steps/Sec: 0.90 + 48%|██████████████████████████████████████████████████▊ | 21999/45917 [8:29:40<7:20:37, 1.11s/it][2025-04-24 01:51:51] (step=0022000) Train Loss: 6.4076, Train Steps/Sec: 0.90 +[2025-04-24 01:51:51] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 01:51:51] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 01:51:51] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.59s/it] +[2025-04-24 01:56:39] Finish Eval in 22000 steps...███████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.11s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 01:57:01] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0022000.pt +[2025-04-24 01:57:03] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0020000.pt + 48%|██████████████████████████████████████████████████▊ | 22024/45917 [8:35:20<7:36:58, 1.15s/it][2025-04-24 01:57:31] (step=0022025) Train Loss: 6.3860, Train Steps/Sec: 0.07 + 48%|██████████████████████████████████████████████████▉ | 22049/45917 [8:35:48<7:23:05, 1.11s/it][2025-04-24 01:57:59] (step=0022050) Train Loss: 6.4335, Train Steps/Sec: 0.90 + 48%|██████████████████████████████████████████████████▉ | 22074/45917 [8:36:16<7:21:55, 1.11s/it][2025-04-24 01:58:27] (step=0022075) Train Loss: 6.4227, Train Steps/Sec: 0.90 + 48%|███████████████████████████████████████████████████ | 22099/45917 [8:36:44<7:15:24, 1.10s/it][2025-04-24 01:58:55] (step=0022100) Train Loss: 6.3763, Train Steps/Sec: 0.90 + 48%|███████████████████████████████████████████████████ | 22124/45917 [8:37:11<7:27:36, 1.13s/it][2025-04-24 01:59:23] (step=0022125) Train Loss: 6.3665, Train Steps/Sec: 0.90 + 48%|███████████████████████████████████████████████████▏ | 22149/45917 [8:37:39<7:19:58, 1.11s/it][2025-04-24 01:59:50] (step=0022150) Train Loss: 6.3554, Train Steps/Sec: 0.90 + 48%|███████████████████████████████████████████████████▏ | 22174/45917 [8:38:07<7:16:19, 1.10s/it][2025-04-24 02:00:18] (step=0022175) Train Loss: 6.3569, Train Steps/Sec: 0.90 + 48%|███████████████████████████████████████████████████▏ | 22199/45917 [8:38:34<7:11:43, 1.09s/it][2025-04-24 02:00:46] (step=0022200) Train Loss: 6.4105, Train Steps/Sec: 0.90 + 48%|███████████████████████████████████████████████████▎ | 22224/45917 [8:39:02<7:22:50, 1.12s/it][2025-04-24 02:01:13] (step=0022225) Train Loss: 6.3752, Train Steps/Sec: 0.90 + 48%|███████████████████████████████████████████████████▎ | 22249/45917 [8:39:30<7:22:01, 1.12s/it][2025-04-24 02:01:41] (step=0022250) Train Loss: 6.4140, Train Steps/Sec: 0.90 + 49%|███████████████████████████████████████████████████▍ | 22274/45917 [8:39:58<7:16:24, 1.11s/it][2025-04-24 02:02:09] (step=0022275) Train Loss: 6.3676, Train Steps/Sec: 0.90 + 49%|███████████████████████████████████████████████████▍ | 22299/45917 [8:40:25<7:11:37, 1.10s/it][2025-04-24 02:02:37] (step=0022300) Train Loss: 6.4027, Train Steps/Sec: 0.90 + 49%|███████████████████████████████████████████████████▌ | 22324/45917 [8:40:53<7:21:40, 1.12s/it][2025-04-24 02:03:04] (step=0022325) Train Loss: 6.3621, Train Steps/Sec: 0.90 + 49%|███████████████████████████████████████████████████▌ | 22349/45917 [8:41:21<7:15:55, 1.11s/it][2025-04-24 02:03:32] (step=0022350) Train Loss: 6.3733, Train Steps/Sec: 0.91 + 49%|███████████████████████████████████████████████████▋ | 22374/45917 [8:41:49<7:17:39, 1.12s/it][2025-04-24 02:04:00] (step=0022375) Train Loss: 6.3605, Train Steps/Sec: 0.90 + 49%|███████████████████████████████████████████████████▋ | 22399/45917 [8:42:16<7:12:49, 1.10s/it][2025-04-24 02:04:28] (step=0022400) Train Loss: 6.4360, Train Steps/Sec: 0.89 + 49%|███████████████████████████████████████████████████▊ | 22424/45917 [8:42:44<7:25:15, 1.14s/it][2025-04-24 02:04:56] (step=0022425) Train Loss: 6.3756, Train Steps/Sec: 0.90 + 49%|███████████████████████████████████████████████████▊ | 22449/45917 [8:43:13<7:15:30, 1.11s/it][2025-04-24 02:05:24] (step=0022450) Train Loss: 6.4143, Train Steps/Sec: 0.90 + 49%|███████████████████████████████████████████████████▉ | 22474/45917 [8:43:40<7:09:32, 1.10s/it][2025-04-24 02:05:51] (step=0022475) Train Loss: 6.3962, Train Steps/Sec: 0.90 + 49%|███████████████████████████████████████████████████▉ | 22499/45917 [8:44:08<7:07:55, 1.10s/it][2025-04-24 02:06:19] (step=0022500) Train Loss: 6.3775, Train Steps/Sec: 0.90 + 49%|███████████████████████████████████████████████████▉ | 22524/45917 [8:44:36<7:16:04, 1.12s/it][2025-04-24 02:06:47] (step=0022525) Train Loss: 6.4170, Train Steps/Sec: 0.90 + 49%|████████████████████████████████████████████████████ | 22549/45917 [8:45:03<7:11:10, 1.11s/it][2025-04-24 02:07:15] (step=0022550) Train Loss: 6.3670, Train Steps/Sec: 0.90 + 49%|████████████████████████████████████████████████████ | 22574/45917 [8:45:31<7:10:53, 1.11s/it][2025-04-24 02:07:42] (step=0022575) Train Loss: 6.3547, Train Steps/Sec: 0.90 + 49%|████████████████████████████████████████████████████▏ | 22599/45917 [8:45:59<7:04:54, 1.09s/it][2025-04-24 02:08:10] (step=0022600) Train Loss: 6.4136, Train Steps/Sec: 0.90 + 49%|████████████████████████████████████████████████████▏ | 22624/45917 [8:46:27<7:18:50, 1.13s/it][2025-04-24 02:08:38] (step=0022625) Train Loss: 6.3927, Train Steps/Sec: 0.90 + 49%|████████████████████████████████████████████████████▎ | 22649/45917 [8:46:54<7:11:55, 1.11s/it][2025-04-24 02:09:06] (step=0022650) Train Loss: 6.3573, Train Steps/Sec: 0.90 + 49%|████████████████████████████████████████████████████▎ | 22674/45917 [8:47:22<7:12:18, 1.12s/it][2025-04-24 02:09:33] (step=0022675) Train Loss: 6.4005, Train Steps/Sec: 0.90 + 49%|████████████████████████████████████████████████████▍ | 22699/45917 [8:47:50<7:03:22, 1.09s/it][2025-04-24 02:10:01] (step=0022700) Train Loss: 6.3756, Train Steps/Sec: 0.90 + 49%|████████████████████████████████████████████████████▍ | 22724/45917 [8:48:18<7:15:53, 1.13s/it][2025-04-24 02:10:29] (step=0022725) Train Loss: 6.3920, Train Steps/Sec: 0.90 + 50%|████████████████████████████████████████████████████▌ | 22749/45917 [8:48:45<7:06:22, 1.10s/it][2025-04-24 02:10:57] (step=0022750) Train Loss: 6.3724, Train Steps/Sec: 0.90 + 50%|████████████████████████████████████████████████████▌ | 22774/45917 [8:49:13<7:06:26, 1.11s/it][2025-04-24 02:11:24] (step=0022775) Train Loss: 6.3713, Train Steps/Sec: 0.90 + 50%|████████████████████████████████████████████████████▋ | 22799/45917 [8:49:41<6:59:49, 1.09s/it][2025-04-24 02:11:52] (step=0022800) Train Loss: 6.4354, Train Steps/Sec: 0.90 + 50%|████████████████████████████████████████████████████▋ | 22824/45917 [8:50:09<7:13:20, 1.13s/it][2025-04-24 02:12:20] (step=0022825) Train Loss: 6.3723, Train Steps/Sec: 0.90 + 50%|████████████████████████████████████████████████████▋ | 22849/45917 [8:50:36<7:03:53, 1.10s/it][2025-04-24 02:12:48] (step=0022850) Train Loss: 6.4122, Train Steps/Sec: 0.90 + 50%|████████████████████████████████████████████████████▊ | 22874/45917 [8:51:04<7:03:10, 1.10s/it][2025-04-24 02:13:15] (step=0022875) Train Loss: 6.4198, Train Steps/Sec: 0.90 + 50%|████████████████████████████████████████████████████▊ | 22899/45917 [8:51:32<6:59:19, 1.09s/it][2025-04-24 02:13:43] (step=0022900) Train Loss: 6.3745, Train Steps/Sec: 0.90 + 50%|████████████████████████████████████████████████████▉ | 22924/45917 [8:52:00<7:12:12, 1.13s/it][2025-04-24 02:14:11] (step=0022925) Train Loss: 6.3674, Train Steps/Sec: 0.90 + 50%|████████████████████████████████████████████████████▉ | 22949/45917 [8:52:27<7:05:36, 1.11s/it][2025-04-24 02:14:38] (step=0022950) Train Loss: 6.3837, Train Steps/Sec: 0.90 + 50%|█████████████████████████████████████████████████████ | 22974/45917 [8:52:55<7:03:46, 1.11s/it][2025-04-24 02:15:06] (step=0022975) Train Loss: 6.3823, Train Steps/Sec: 0.90 + 50%|█████████████████████████████████████████████████████ | 22999/45917 [8:53:23<7:00:05, 1.10s/it][2025-04-24 02:15:34] (step=0023000) Train Loss: 6.4245, Train Steps/Sec: 0.90 + 50%|█████████████████████████████████████████████████████▏ | 23024/45917 [8:53:51<7:10:24, 1.13s/it][2025-04-24 02:16:02] (step=0023025) Train Loss: 6.3543, Train Steps/Sec: 0.90 + 50%|█████████████████████████████████████████████████████▏ | 23049/45917 [8:54:18<7:02:08, 1.11s/it][2025-04-24 02:16:29] (step=0023050) Train Loss: 6.3893, Train Steps/Sec: 0.90 + 50%|█████████████████████████████████████████████████████▎ | 23074/45917 [8:54:46<6:59:49, 1.10s/it][2025-04-24 02:16:57] (step=0023075) Train Loss: 6.3777, Train Steps/Sec: 0.91 + 50%|█████████████████████████████████████████████████████▎ | 23099/45917 [8:55:14<6:56:05, 1.09s/it][2025-04-24 02:17:25] (step=0023100) Train Loss: 6.4188, Train Steps/Sec: 0.90 + 50%|█████████████████████████████████████████████████████▍ | 23124/45917 [8:55:41<7:06:17, 1.12s/it][2025-04-24 02:17:52] (step=0023125) Train Loss: 6.3391, Train Steps/Sec: 0.90 + 50%|█████████████████████████████████████████████████████▍ | 23149/45917 [8:56:09<6:59:42, 1.11s/it][2025-04-24 02:18:20] (step=0023150) Train Loss: 6.4083, Train Steps/Sec: 0.91 + 50%|█████████████████████████████████████████████████████▍ | 23174/45917 [8:56:36<6:55:18, 1.10s/it][2025-04-24 02:18:48] (step=0023175) Train Loss: 6.4068, Train Steps/Sec: 0.91 + 51%|█████████████████████████████████████████████████████▌ | 23199/45917 [8:57:04<6:56:06, 1.10s/it][2025-04-24 02:19:16] (step=0023200) Train Loss: 6.4079, Train Steps/Sec: 0.90 + 51%|█████████████████████████████████████████████████████▌ | 23224/45917 [8:57:32<7:05:01, 1.12s/it][2025-04-24 02:19:43] (step=0023225) Train Loss: 6.3575, Train Steps/Sec: 0.90 + 51%|█████████████████████████████████████████████████████▋ | 23249/45917 [8:58:00<6:57:39, 1.11s/it][2025-04-24 02:20:11] (step=0023250) Train Loss: 6.3846, Train Steps/Sec: 0.90 + 51%|█████████████████████████████████████████████████████▋ | 23274/45917 [8:58:27<6:56:25, 1.10s/it][2025-04-24 02:20:39] (step=0023275) Train Loss: 6.3639, Train Steps/Sec: 0.91 + 51%|█████████████████████████████████████████████████████▊ | 23299/45917 [8:58:55<6:53:34, 1.10s/it][2025-04-24 02:21:06] (step=0023300) Train Loss: 6.3610, Train Steps/Sec: 0.90 + 51%|█████████████████████████████████████████████████████▊ | 23324/45917 [8:59:23<7:05:28, 1.13s/it][2025-04-24 02:21:34] (step=0023325) Train Loss: 6.4242, Train Steps/Sec: 0.90 + 51%|█████████████████████████████████████████████████████▉ | 23349/45917 [8:59:50<6:56:13, 1.11s/it][2025-04-24 02:22:02] (step=0023350) Train Loss: 6.3597, Train Steps/Sec: 0.90 + 51%|█████████████████████████████████████████████████████▉ | 23374/45917 [9:00:18<6:53:17, 1.10s/it][2025-04-24 02:22:29] (step=0023375) Train Loss: 6.3490, Train Steps/Sec: 0.90 + 51%|██████████████████████████████████████████████████████ | 23399/45917 [9:00:46<6:51:45, 1.10s/it][2025-04-24 02:22:57] (step=0023400) Train Loss: 6.4008, Train Steps/Sec: 0.90 + 51%|██████████████████████████████████████████████████████ | 23424/45917 [9:01:14<7:04:17, 1.13s/it][2025-04-24 02:23:25] (step=0023425) Train Loss: 6.3997, Train Steps/Sec: 0.90 + 51%|██████████████████████████████████████████████████████▏ | 23449/45917 [9:01:41<6:58:05, 1.12s/it][2025-04-24 02:23:53] (step=0023450) Train Loss: 6.3973, Train Steps/Sec: 0.90 + 51%|██████████████████████████████████████████████████████▏ | 23474/45917 [9:02:09<6:51:58, 1.10s/it][2025-04-24 02:24:20] (step=0023475) Train Loss: 6.4097, Train Steps/Sec: 0.90 + 51%|██████████████████████████████████████████████████████▏ | 23499/45917 [9:02:37<6:49:50, 1.10s/it][2025-04-24 02:24:48] (step=0023500) Train Loss: 6.3769, Train Steps/Sec: 0.90 + 51%|██████████████████████████████████████████████████████▎ | 23524/45917 [9:03:05<7:03:31, 1.13s/it][2025-04-24 02:25:16] (step=0023525) Train Loss: 6.3433, Train Steps/Sec: 0.90 + 51%|██████████████████████████████████████████████████████▎ | 23549/45917 [9:03:32<6:53:51, 1.11s/it][2025-04-24 02:25:44] (step=0023550) Train Loss: 6.3647, Train Steps/Sec: 0.90 + 51%|██████████████████████████████████████████████████████▍ | 23574/45917 [9:04:00<6:50:40, 1.10s/it][2025-04-24 02:26:11] (step=0023575) Train Loss: 6.4145, Train Steps/Sec: 0.90 + 51%|██████████████████████████████████████████████████████▍ | 23599/45917 [9:04:28<6:47:46, 1.10s/it][2025-04-24 02:26:39] (step=0023600) Train Loss: 6.4279, Train Steps/Sec: 0.90 + 51%|██████████████████████████████████████████████████████▌ | 23624/45917 [9:04:56<7:00:15, 1.13s/it][2025-04-24 02:27:07] (step=0023625) Train Loss: 6.3645, Train Steps/Sec: 0.90 + 52%|██████████████████████████████████████████████████████▌ | 23649/45917 [9:05:23<6:50:02, 1.10s/it][2025-04-24 02:27:35] (step=0023650) Train Loss: 6.4204, Train Steps/Sec: 0.90 + 52%|██████████████████████████████████████████████████████▋ | 23674/45917 [9:05:51<6:49:44, 1.11s/it][2025-04-24 02:28:02] (step=0023675) Train Loss: 6.4195, Train Steps/Sec: 0.90 + 52%|██████████████████████████████████████████████████████▋ | 23699/45917 [9:06:19<6:44:43, 1.09s/it][2025-04-24 02:28:30] (step=0023700) Train Loss: 6.3396, Train Steps/Sec: 0.90 + 52%|██████████████████████████████████████████████████████▊ | 23724/45917 [9:06:47<6:56:51, 1.13s/it][2025-04-24 02:28:58] (step=0023725) Train Loss: 6.3895, Train Steps/Sec: 0.90 + 52%|██████████████████████████████████████████████████████▊ | 23749/45917 [9:07:14<6:51:47, 1.11s/it][2025-04-24 02:29:26] (step=0023750) Train Loss: 6.3929, Train Steps/Sec: 0.90 + 52%|██████████████████████████████████████████████████████▉ | 23774/45917 [9:07:42<6:46:30, 1.10s/it][2025-04-24 02:29:53] (step=0023775) Train Loss: 6.3795, Train Steps/Sec: 0.91 + 52%|██████████████████████████████████████████████████████▉ | 23799/45917 [9:08:10<6:43:48, 1.10s/it][2025-04-24 02:30:21] (step=0023800) Train Loss: 6.3987, Train Steps/Sec: 0.90 + 52%|██████████████████████████████████████████████████████▉ | 23824/45917 [9:08:38<6:55:33, 1.13s/it][2025-04-24 02:30:49] (step=0023825) Train Loss: 6.3580, Train Steps/Sec: 0.90 + 52%|███████████████████████████████████████████████████████ | 23849/45917 [9:09:05<6:47:11, 1.11s/it][2025-04-24 02:31:17] (step=0023850) Train Loss: 6.3590, Train Steps/Sec: 0.90 + 52%|███████████████████████████████████████████████████████ | 23874/45917 [9:09:33<6:45:09, 1.10s/it][2025-04-24 02:31:44] (step=0023875) Train Loss: 6.4097, Train Steps/Sec: 0.90 + 52%|███████████████████████████████████████████████████████▏ | 23899/45917 [9:10:01<6:39:25, 1.09s/it][2025-04-24 02:32:12] (step=0023900) Train Loss: 6.3602, Train Steps/Sec: 0.90 + 52%|███████████████████████████████████████████████████████▏ | 23924/45917 [9:10:28<6:52:55, 1.13s/it][2025-04-24 02:32:40] (step=0023925) Train Loss: 6.3734, Train Steps/Sec: 0.90 + 52%|███████████████████████████████████████████████████████▎ | 23949/45917 [9:10:56<6:44:09, 1.10s/it][2025-04-24 02:33:07] (step=0023950) Train Loss: 6.3986, Train Steps/Sec: 0.90 + 52%|███████████████████████████████████████████████████████▎ | 23974/45917 [9:11:24<6:40:29, 1.10s/it][2025-04-24 02:33:35] (step=0023975) Train Loss: 6.4038, Train Steps/Sec: 0.90 + 52%|███████████████████████████████████████████████████████▍ | 23999/45917 [9:11:52<6:42:16, 1.10s/it][2025-04-24 02:34:03] (step=0024000) Train Loss: 6.4598, Train Steps/Sec: 0.89 +[2025-04-24 02:34:03] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 02:34:03] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 02:34:03] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:59<00:00, 59.68s/it] +[2025-04-24 02:38:51] Finish Eval in 24000 steps...█████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.12s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 02:39:12] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0024000.pt +[2025-04-24 02:39:14] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0022000.pt + 52%|███████████████████████████████████████████████████████▍ | 24024/45917 [9:17:31<6:53:36, 1.13s/it][2025-04-24 02:39:42] (step=0024025) Train Loss: 6.3792, Train Steps/Sec: 0.07 + 52%|███████████████████████████████████████████████████████▌ | 24049/45917 [9:17:59<6:45:23, 1.11s/it][2025-04-24 02:40:10] (step=0024050) Train Loss: 6.3697, Train Steps/Sec: 0.90 + 52%|███████████████████████████████████████████████████████▌ | 24074/45917 [9:18:27<6:41:38, 1.10s/it][2025-04-24 02:40:38] (step=0024075) Train Loss: 6.4473, Train Steps/Sec: 0.90 + 52%|███████████████████████████████████████████████████████▋ | 24099/45917 [9:18:54<6:40:12, 1.10s/it][2025-04-24 02:41:06] (step=0024100) Train Loss: 6.4237, Train Steps/Sec: 0.90 + 53%|███████████████████████████████████████████████████████▋ | 24124/45917 [9:19:22<6:51:03, 1.13s/it][2025-04-24 02:41:33] (step=0024125) Train Loss: 6.3575, Train Steps/Sec: 0.90 + 53%|███████████████████████████████████████████████████████▋ | 24149/45917 [9:19:50<6:43:39, 1.11s/it][2025-04-24 02:42:01] (step=0024150) Train Loss: 6.3688, Train Steps/Sec: 0.90 + 53%|███████████████████████████████████████████████████████▊ | 24174/45917 [9:20:18<6:37:25, 1.10s/it][2025-04-24 02:42:29] (step=0024175) Train Loss: 6.4013, Train Steps/Sec: 0.90 + 53%|███████████████████████████████████████████████████████▊ | 24199/45917 [9:20:45<6:35:05, 1.09s/it][2025-04-24 02:42:57] (step=0024200) Train Loss: 6.3664, Train Steps/Sec: 0.90 + 53%|███████████████████████████████████████████████████████▉ | 24224/45917 [9:21:13<6:45:35, 1.12s/it][2025-04-24 02:43:24] (step=0024225) Train Loss: 6.4143, Train Steps/Sec: 0.90 + 53%|███████████████████████████████████████████████████████▉ | 24249/45917 [9:21:41<6:36:20, 1.10s/it][2025-04-24 02:43:52] (step=0024250) Train Loss: 6.3896, Train Steps/Sec: 0.90 + 53%|████████████████████████████████████████████████████████ | 24274/45917 [9:22:09<6:36:48, 1.10s/it][2025-04-24 02:44:20] (step=0024275) Train Loss: 6.4034, Train Steps/Sec: 0.91 + 53%|████████████████████████████████████████████████████████ | 24299/45917 [9:22:36<6:34:05, 1.09s/it][2025-04-24 02:44:48] (step=0024300) Train Loss: 6.4040, Train Steps/Sec: 0.89 + 53%|████████████████████████████████████████████████████████▏ | 24324/45917 [9:23:04<6:47:17, 1.13s/it][2025-04-24 02:45:15] (step=0024325) Train Loss: 6.3828, Train Steps/Sec: 0.90 + 53%|████████████████████████████████████████████████████████▏ | 24349/45917 [9:23:32<6:41:33, 1.12s/it][2025-04-24 02:45:43] (step=0024350) Train Loss: 6.4303, Train Steps/Sec: 0.90 + 53%|████████████████████████████████████████████████████████▎ | 24374/45917 [9:24:00<6:35:39, 1.10s/it][2025-04-24 02:46:11] (step=0024375) Train Loss: 6.3843, Train Steps/Sec: 0.91 + 53%|████████████████████████████████████████████████████████▎ | 24399/45917 [9:24:27<6:35:36, 1.10s/it][2025-04-24 02:46:39] (step=0024400) Train Loss: 6.4265, Train Steps/Sec: 0.89 + 53%|████████████████████████████████████████████████████████▍ | 24424/45917 [9:24:55<6:43:49, 1.13s/it][2025-04-24 02:47:06] (step=0024425) Train Loss: 6.3829, Train Steps/Sec: 0.90 + 53%|████████████████████████████████████████████████████████▍ | 24449/45917 [9:25:23<6:37:16, 1.11s/it][2025-04-24 02:47:34] (step=0024450) Train Loss: 6.3917, Train Steps/Sec: 0.90 + 53%|████████████████████████████████████████████████████████▍ | 24474/45917 [9:25:51<6:32:28, 1.10s/it][2025-04-24 02:48:02] (step=0024475) Train Loss: 6.4017, Train Steps/Sec: 0.90 + 53%|████████████████████████████████████████████████████████▌ | 24499/45917 [9:26:18<6:31:43, 1.10s/it][2025-04-24 02:48:30] (step=0024500) Train Loss: 6.3397, Train Steps/Sec: 0.90 + 53%|████████████████████████████████████████████████████████▌ | 24524/45917 [9:26:46<6:44:17, 1.13s/it][2025-04-24 02:48:57] (step=0024525) Train Loss: 6.3664, Train Steps/Sec: 0.90 + 53%|████████████████████████████████████████████████████████▋ | 24549/45917 [9:27:14<6:40:20, 1.12s/it][2025-04-24 02:49:25] (step=0024550) Train Loss: 6.4211, Train Steps/Sec: 0.90 + 54%|████████████████████████████████████████████████████████▋ | 24574/45917 [9:27:41<6:30:13, 1.10s/it][2025-04-24 02:49:53] (step=0024575) Train Loss: 6.4027, Train Steps/Sec: 0.91 + 54%|████████████████████████████████████████████████████████▊ | 24599/45917 [9:28:09<6:29:02, 1.09s/it][2025-04-24 02:50:21] (step=0024600) Train Loss: 6.3614, Train Steps/Sec: 0.90 + 54%|████████████████████████████████████████████████████████▊ | 24624/45917 [9:28:37<6:45:13, 1.14s/it][2025-04-24 02:50:48] (step=0024625) Train Loss: 6.3567, Train Steps/Sec: 0.90 + 54%|████████████████████████████████████████████████████████▉ | 24649/45917 [9:29:05<6:33:08, 1.11s/it][2025-04-24 02:51:16] (step=0024650) Train Loss: 6.3979, Train Steps/Sec: 0.90 + 54%|████████████████████████████████████████████████████████▉ | 24674/45917 [9:29:33<6:32:34, 1.11s/it][2025-04-24 02:51:44] (step=0024675) Train Loss: 6.4398, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████ | 24699/45917 [9:30:00<6:26:53, 1.09s/it][2025-04-24 02:52:12] (step=0024700) Train Loss: 6.4173, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████ | 24724/45917 [9:30:28<6:37:32, 1.13s/it][2025-04-24 02:52:39] (step=0024725) Train Loss: 6.3413, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████▏ | 24749/45917 [9:30:56<6:28:37, 1.10s/it][2025-04-24 02:53:07] (step=0024750) Train Loss: 6.4078, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████▏ | 24774/45917 [9:31:23<6:27:36, 1.10s/it][2025-04-24 02:53:35] (step=0024775) Train Loss: 6.3473, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████▏ | 24799/45917 [9:31:51<6:21:55, 1.09s/it][2025-04-24 02:54:02] (step=0024800) Train Loss: 6.4087, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████▎ | 24824/45917 [9:32:19<6:35:39, 1.13s/it][2025-04-24 02:54:30] (step=0024825) Train Loss: 6.3836, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████▎ | 24849/45917 [9:32:47<6:30:18, 1.11s/it][2025-04-24 02:54:58] (step=0024850) Train Loss: 6.3530, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████▍ | 24874/45917 [9:33:14<6:24:04, 1.10s/it][2025-04-24 02:55:26] (step=0024875) Train Loss: 6.3988, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████▍ | 24899/45917 [9:33:42<6:22:32, 1.09s/it][2025-04-24 02:55:53] (step=0024900) Train Loss: 6.3630, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████▌ | 24924/45917 [9:34:10<6:34:47, 1.13s/it][2025-04-24 02:56:21] (step=0024925) Train Loss: 6.3761, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████▌ | 24949/45917 [9:34:38<6:29:56, 1.12s/it][2025-04-24 02:56:49] (step=0024950) Train Loss: 6.3819, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████ | 24974/45917 [9:35:17<13:27:04, 2.31s/it][2025-04-24 02:57:29] (step=0024975) Train Loss: 6.4199, Train Steps/Sec: 0.63 + 54%|█████████████████████████████████████████████████████████▋ | 24999/45917 [9:35:45<6:21:38, 1.09s/it][2025-04-24 02:57:57] (step=0025000) Train Loss: 6.4320, Train Steps/Sec: 0.90 + 54%|█████████████████████████████████████████████████████████▊ | 25024/45917 [9:36:13<6:30:14, 1.12s/it][2025-04-24 02:58:24] (step=0025025) Train Loss: 6.3881, Train Steps/Sec: 0.90 + 55%|█████████████████████████████████████████████████████████▊ | 25049/45917 [9:36:41<6:28:06, 1.12s/it][2025-04-24 02:58:52] (step=0025050) Train Loss: 6.3588, Train Steps/Sec: 0.90 + 55%|█████████████████████████████████████████████████████████▉ | 25074/45917 [9:37:08<6:22:26, 1.10s/it][2025-04-24 02:59:20] (step=0025075) Train Loss: 6.3843, Train Steps/Sec: 0.91 + 55%|█████████████████████████████████████████████████████████▉ | 25099/45917 [9:37:36<6:20:29, 1.10s/it][2025-04-24 02:59:48] (step=0025100) Train Loss: 6.3859, Train Steps/Sec: 0.90 + 55%|█████████████████████████████████████████████████████████▉ | 25124/45917 [9:38:04<6:30:46, 1.13s/it][2025-04-24 03:00:15] (step=0025125) Train Loss: 6.3702, Train Steps/Sec: 0.90 + 55%|██████████████████████████████████████████████████████████ | 25149/45917 [9:38:32<6:30:04, 1.13s/it][2025-04-24 03:00:43] (step=0025150) Train Loss: 6.4094, Train Steps/Sec: 0.90 + 55%|██████████████████████████████████████████████████████████ | 25174/45917 [9:39:06<6:38:02, 1.15s/it][2025-04-24 03:01:17] (step=0025175) Train Loss: 6.3768, Train Steps/Sec: 0.74 + 55%|██████████████████████████████████████████████████████████▏ | 25199/45917 [9:39:34<6:20:31, 1.10s/it][2025-04-24 03:01:45] (step=0025200) Train Loss: 6.4101, Train Steps/Sec: 0.90 + 55%|█████████████████████████████████████████████████████████▋ | 25224/45917 [9:40:08<11:47:02, 2.05s/it][2025-04-24 03:02:19] (step=0025225) Train Loss: 6.3563, Train Steps/Sec: 0.74 + 55%|██████████████████████████████████████████████████████████▎ | 25249/45917 [9:40:35<6:18:24, 1.10s/it][2025-04-24 03:02:47] (step=0025250) Train Loss: 6.4023, Train Steps/Sec: 0.90 + 55%|█████████████████████████████████████████████████████████▊ | 25274/45917 [9:41:09<13:44:31, 2.40s/it][2025-04-24 03:03:20] (step=0025275) Train Loss: 6.3797, Train Steps/Sec: 0.74 + 55%|██████████████████████████████████████████████████████████▍ | 25299/45917 [9:41:37<6:19:06, 1.10s/it][2025-04-24 03:03:48] (step=0025300) Train Loss: 6.3220, Train Steps/Sec: 0.90 + 55%|██████████████████████████████████████████████████████████▍ | 25324/45917 [9:42:05<6:29:00, 1.13s/it][2025-04-24 03:04:16] (step=0025325) Train Loss: 6.3798, Train Steps/Sec: 0.90 + 55%|██████████████████████████████████████████████████████████▌ | 25349/45917 [9:42:32<6:20:41, 1.11s/it][2025-04-24 03:04:44] (step=0025350) Train Loss: 6.3803, Train Steps/Sec: 0.90 + 55%|██████████████████████████████████████████████████████████▌ | 25374/45917 [9:43:00<6:17:12, 1.10s/it][2025-04-24 03:05:11] (step=0025375) Train Loss: 6.3584, Train Steps/Sec: 0.90 + 55%|██████████████████████████████████████████████████████████▋ | 25399/45917 [9:43:28<6:19:33, 1.11s/it][2025-04-24 03:05:39] (step=0025400) Train Loss: 6.3969, Train Steps/Sec: 0.90 + 55%|██████████████████████████████████████████████████████████▋ | 25424/45917 [9:43:56<6:25:23, 1.13s/it][2025-04-24 03:06:07] (step=0025425) Train Loss: 6.3421, Train Steps/Sec: 0.90 + 55%|██████████████████████████████████████████████████████████▋ | 25449/45917 [9:44:23<6:18:13, 1.11s/it][2025-04-24 03:06:35] (step=0025450) Train Loss: 6.4065, Train Steps/Sec: 0.91 + 55%|██████████████████████████████████████████████████████████▊ | 25474/45917 [9:44:51<6:16:11, 1.10s/it][2025-04-24 03:07:02] (step=0025475) Train Loss: 6.3679, Train Steps/Sec: 0.90 + 56%|██████████████████████████████████████████████████████████▊ | 25499/45917 [9:45:19<6:13:01, 1.10s/it][2025-04-24 03:07:30] (step=0025500) Train Loss: 6.3674, Train Steps/Sec: 0.90 + 56%|██████████████████████████████████████████████████████████▉ | 25524/45917 [9:45:47<6:23:55, 1.13s/it][2025-04-24 03:07:58] (step=0025525) Train Loss: 6.4193, Train Steps/Sec: 0.90 + 56%|██████████████████████████████████████████████████████████▉ | 25549/45917 [9:46:14<6:16:28, 1.11s/it][2025-04-24 03:08:25] (step=0025550) Train Loss: 6.3370, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████ | 25574/45917 [9:46:42<6:12:42, 1.10s/it][2025-04-24 03:08:53] (step=0025575) Train Loss: 6.3410, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████ | 25599/45917 [9:47:10<6:10:45, 1.09s/it][2025-04-24 03:09:21] (step=0025600) Train Loss: 6.3596, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▏ | 25624/45917 [9:47:38<6:19:13, 1.12s/it][2025-04-24 03:09:49] (step=0025625) Train Loss: 6.4144, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▏ | 25649/45917 [9:48:05<6:10:11, 1.10s/it][2025-04-24 03:10:16] (step=0025650) Train Loss: 6.3570, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▎ | 25674/45917 [9:48:33<6:10:40, 1.10s/it][2025-04-24 03:10:44] (step=0025675) Train Loss: 6.3760, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▎ | 25699/45917 [9:49:01<6:09:40, 1.10s/it][2025-04-24 03:11:12] (step=0025700) Train Loss: 6.3526, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▍ | 25724/45917 [9:49:28<6:19:19, 1.13s/it][2025-04-24 03:11:40] (step=0025725) Train Loss: 6.3718, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▍ | 25749/45917 [9:49:56<6:14:22, 1.11s/it][2025-04-24 03:12:07] (step=0025750) Train Loss: 6.4198, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▍ | 25774/45917 [9:50:24<6:11:31, 1.11s/it][2025-04-24 03:12:35] (step=0025775) Train Loss: 6.3585, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▌ | 25799/45917 [9:50:52<6:12:03, 1.11s/it][2025-04-24 03:13:03] (step=0025800) Train Loss: 6.4289, Train Steps/Sec: 0.89 + 56%|███████████████████████████████████████████████████████████▌ | 25824/45917 [9:51:20<6:18:07, 1.13s/it][2025-04-24 03:13:31] (step=0025825) Train Loss: 6.3546, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▋ | 25849/45917 [9:51:48<6:13:55, 1.12s/it][2025-04-24 03:13:59] (step=0025850) Train Loss: 6.3828, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▋ | 25874/45917 [9:52:15<6:08:10, 1.10s/it][2025-04-24 03:14:27] (step=0025875) Train Loss: 6.3932, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▊ | 25899/45917 [9:52:43<6:06:45, 1.10s/it][2025-04-24 03:14:55] (step=0025900) Train Loss: 6.3980, Train Steps/Sec: 0.90 + 56%|███████████████████████████████████████████████████████████▊ | 25924/45917 [9:53:18<6:33:04, 1.18s/it][2025-04-24 03:15:29] (step=0025925) Train Loss: 6.3470, Train Steps/Sec: 0.73 + 57%|███████████████████████████████████████████████████████████▉ | 25949/45917 [9:53:45<6:11:04, 1.11s/it][2025-04-24 03:15:57] (step=0025950) Train Loss: 6.3802, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████▉ | 25974/45917 [9:54:13<6:01:55, 1.09s/it][2025-04-24 03:16:24] (step=0025975) Train Loss: 6.4182, Train Steps/Sec: 0.90 + 57%|████████████████████████████████████████████████████████████ | 25999/45917 [9:54:48<6:05:53, 1.10s/it][2025-04-24 03:16:59] (step=0026000) Train Loss: 6.3488, Train Steps/Sec: 0.72 +[2025-04-24 03:16:59] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 03:16:59] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 03:16:59] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.68s/it] +[2025-04-24 03:21:48] Finish Eval in 26000 steps...█████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.25s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 03:22:10] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0026000.pt +[2025-04-24 03:22:13] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0024000.pt + 57%|███████████████████████████████████████████████████████████▌ | 26024/45917 [10:00:30<6:21:24, 1.15s/it][2025-04-24 03:22:41] (step=0026025) Train Loss: 6.4557, Train Steps/Sec: 0.07 + 57%|███████████████████████████████████████████████████████████▌ | 26049/45917 [10:00:57<6:08:36, 1.11s/it][2025-04-24 03:23:08] (step=0026050) Train Loss: 6.3895, Train Steps/Sec: 0.91 + 57%|███████████████████████████████████████████████████████████▌ | 26074/45917 [10:01:25<6:06:20, 1.11s/it][2025-04-24 03:23:36] (step=0026075) Train Loss: 6.4637, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████▋ | 26099/45917 [10:01:52<6:00:14, 1.09s/it][2025-04-24 03:24:04] (step=0026100) Train Loss: 6.3989, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████▋ | 26124/45917 [10:02:20<6:12:11, 1.13s/it][2025-04-24 03:24:31] (step=0026125) Train Loss: 6.3550, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████▊ | 26149/45917 [10:02:48<6:02:34, 1.10s/it][2025-04-24 03:24:59] (step=0026150) Train Loss: 6.3499, Train Steps/Sec: 0.89 + 57%|███████████████████████████████████████████████████████████▊ | 26174/45917 [10:03:16<6:03:52, 1.11s/it][2025-04-24 03:25:27] (step=0026175) Train Loss: 6.4043, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████▉ | 26199/45917 [10:03:44<6:02:17, 1.10s/it][2025-04-24 03:25:55] (step=0026200) Train Loss: 6.3996, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████▉ | 26224/45917 [10:04:11<6:09:19, 1.13s/it][2025-04-24 03:26:22] (step=0026225) Train Loss: 6.3766, Train Steps/Sec: 0.91 + 57%|████████████████████████████████████████████████████████████ | 26249/45917 [10:04:39<6:03:54, 1.11s/it][2025-04-24 03:26:50] (step=0026250) Train Loss: 6.3865, Train Steps/Sec: 0.90 + 57%|████████████████████████████████████████████████████████████ | 26274/45917 [10:05:07<6:03:03, 1.11s/it][2025-04-24 03:27:18] (step=0026275) Train Loss: 6.3558, Train Steps/Sec: 0.90 + 57%|████████████████████████████████████████████████████████████▏ | 26299/45917 [10:05:34<5:56:34, 1.09s/it][2025-04-24 03:27:45] (step=0026300) Train Loss: 6.3995, Train Steps/Sec: 0.90 + 57%|████████████████████████████████████████████████████████████▏ | 26324/45917 [10:06:02<6:06:55, 1.12s/it][2025-04-24 03:28:13] (step=0026325) Train Loss: 6.3893, Train Steps/Sec: 0.91 + 57%|████████████████████████████████████████████████████████████▎ | 26349/45917 [10:06:30<6:01:53, 1.11s/it][2025-04-24 03:28:41] (step=0026350) Train Loss: 6.3948, Train Steps/Sec: 0.90 + 57%|████████████████████████████████████████████████████████████▎ | 26374/45917 [10:06:57<5:58:40, 1.10s/it][2025-04-24 03:29:09] (step=0026375) Train Loss: 6.3611, Train Steps/Sec: 0.90 + 57%|████████████████████████████████████████████████████████████▎ | 26399/45917 [10:07:25<5:57:46, 1.10s/it][2025-04-24 03:29:36] (step=0026400) Train Loss: 6.3507, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████▍ | 26424/45917 [10:07:53<6:07:42, 1.13s/it][2025-04-24 03:30:04] (step=0026425) Train Loss: 6.3799, Train Steps/Sec: 0.91 + 58%|████████████████████████████████████████████████████████████▍ | 26449/45917 [10:08:20<6:01:24, 1.11s/it][2025-04-24 03:30:32] (step=0026450) Train Loss: 6.3798, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████▌ | 26474/45917 [10:08:48<5:54:47, 1.09s/it][2025-04-24 03:30:59] (step=0026475) Train Loss: 6.3710, Train Steps/Sec: 0.91 + 58%|████████████████████████████████████████████████████████████▌ | 26499/45917 [10:09:16<5:55:10, 1.10s/it][2025-04-24 03:31:27] (step=0026500) Train Loss: 6.4230, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████▋ | 26524/45917 [10:09:43<6:01:53, 1.12s/it][2025-04-24 03:31:55] (step=0026525) Train Loss: 6.4184, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████▋ | 26549/45917 [10:10:11<6:00:03, 1.12s/it][2025-04-24 03:32:22] (step=0026550) Train Loss: 6.3668, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████▊ | 26574/45917 [10:10:39<5:55:18, 1.10s/it][2025-04-24 03:32:50] (step=0026575) Train Loss: 6.3541, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████▊ | 26599/45917 [10:11:06<5:51:24, 1.09s/it][2025-04-24 03:33:18] (step=0026600) Train Loss: 6.3572, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████▉ | 26624/45917 [10:11:34<6:03:08, 1.13s/it][2025-04-24 03:33:45] (step=0026625) Train Loss: 6.3652, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████▉ | 26649/45917 [10:12:02<5:54:40, 1.10s/it][2025-04-24 03:34:13] (step=0026650) Train Loss: 6.4297, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████▉ | 26674/45917 [10:12:30<5:55:09, 1.11s/it][2025-04-24 03:34:41] (step=0026675) Train Loss: 6.3530, Train Steps/Sec: 0.90 + 58%|█████████████████████████████████████████████████████████████ | 26699/45917 [10:12:57<5:49:27, 1.09s/it][2025-04-24 03:35:09] (step=0026700) Train Loss: 6.4083, Train Steps/Sec: 0.90 + 58%|█████████████████████████████████████████████████████████████ | 26724/45917 [10:13:25<6:03:22, 1.14s/it][2025-04-24 03:35:36] (step=0026725) Train Loss: 6.3946, Train Steps/Sec: 0.90 + 58%|█████████████████████████████████████████████████████████████▏ | 26749/45917 [10:13:53<5:56:32, 1.12s/it][2025-04-24 03:36:04] (step=0026750) Train Loss: 6.4078, Train Steps/Sec: 0.90 + 58%|█████████████████████████████████████████████████████████████▏ | 26774/45917 [10:14:21<5:49:32, 1.10s/it][2025-04-24 03:36:32] (step=0026775) Train Loss: 6.4284, Train Steps/Sec: 0.90 + 58%|█████████████████████████████████████████████████████████████▎ | 26799/45917 [10:14:48<5:46:23, 1.09s/it][2025-04-24 03:36:59] (step=0026800) Train Loss: 6.3674, Train Steps/Sec: 0.90 + 58%|█████████████████████████████████████████████████████████████▎ | 26824/45917 [10:15:16<6:00:23, 1.13s/it][2025-04-24 03:37:27] (step=0026825) Train Loss: 6.3738, Train Steps/Sec: 0.90 + 58%|█████████████████████████████████████████████████████████████▍ | 26849/45917 [10:15:44<5:53:03, 1.11s/it][2025-04-24 03:37:55] (step=0026850) Train Loss: 6.3649, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████▍ | 26874/45917 [10:16:11<5:50:43, 1.11s/it][2025-04-24 03:38:23] (step=0026875) Train Loss: 6.3877, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████▌ | 26899/45917 [10:16:39<5:52:47, 1.11s/it][2025-04-24 03:38:51] (step=0026900) Train Loss: 6.3799, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████▌ | 26924/45917 [10:17:07<5:58:49, 1.13s/it][2025-04-24 03:39:18] (step=0026925) Train Loss: 6.3680, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████▋ | 26949/45917 [10:17:35<5:52:43, 1.12s/it][2025-04-24 03:39:46] (step=0026950) Train Loss: 6.3608, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████▋ | 26974/45917 [10:18:03<5:50:13, 1.11s/it][2025-04-24 03:40:14] (step=0026975) Train Loss: 6.3854, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████▋ | 26999/45917 [10:18:31<5:46:04, 1.10s/it][2025-04-24 03:40:42] (step=0027000) Train Loss: 6.3523, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████▊ | 27024/45917 [10:18:58<5:53:05, 1.12s/it][2025-04-24 03:41:10] (step=0027025) Train Loss: 6.3636, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████▊ | 27049/45917 [10:19:26<5:50:25, 1.11s/it][2025-04-24 03:41:37] (step=0027050) Train Loss: 6.3867, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████▉ | 27074/45917 [10:19:54<5:46:27, 1.10s/it][2025-04-24 03:42:05] (step=0027075) Train Loss: 6.3877, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████▉ | 27099/45917 [10:20:22<5:44:30, 1.10s/it][2025-04-24 03:42:33] (step=0027100) Train Loss: 6.3817, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████ | 27124/45917 [10:20:49<5:52:43, 1.13s/it][2025-04-24 03:43:01] (step=0027125) Train Loss: 6.4274, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████ | 27149/45917 [10:21:17<5:48:19, 1.11s/it][2025-04-24 03:43:28] (step=0027150) Train Loss: 6.3587, Train Steps/Sec: 0.91 + 59%|██████████████████████████████████████████████████████████████▏ | 27174/45917 [10:21:45<5:42:12, 1.10s/it][2025-04-24 03:43:56] (step=0027175) Train Loss: 6.3980, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████▏ | 27199/45917 [10:22:12<5:42:57, 1.10s/it][2025-04-24 03:44:24] (step=0027200) Train Loss: 6.3777, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████▎ | 27224/45917 [10:22:40<5:51:59, 1.13s/it][2025-04-24 03:44:51] (step=0027225) Train Loss: 6.3865, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████▎ | 27249/45917 [10:23:08<5:45:50, 1.11s/it][2025-04-24 03:45:19] (step=0027250) Train Loss: 6.3609, Train Steps/Sec: 0.91 + 59%|██████████████████████████████████████████████████████████████▎ | 27274/45917 [10:23:36<5:44:17, 1.11s/it][2025-04-24 03:45:47] (step=0027275) Train Loss: 6.4083, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████▍ | 27299/45917 [10:24:03<5:42:51, 1.10s/it][2025-04-24 03:46:15] (step=0027300) Train Loss: 6.3930, Train Steps/Sec: 0.90 + 60%|██████████████████████████████████████████████████████████████▍ | 27324/45917 [10:24:31<5:45:29, 1.11s/it][2025-04-24 03:46:42] (step=0027325) Train Loss: 6.3953, Train Steps/Sec: 0.91 + 60%|██████████████████████████████████████████████████████████████▌ | 27349/45917 [10:24:59<5:42:01, 1.11s/it][2025-04-24 03:47:10] (step=0027350) Train Loss: 6.4131, Train Steps/Sec: 0.90 + 60%|██████████████████████████████████████████████████████████████▌ | 27374/45917 [10:25:26<5:37:39, 1.09s/it][2025-04-24 03:47:37] (step=0027375) Train Loss: 6.4050, Train Steps/Sec: 0.91 + 60%|██████████████████████████████████████████████████████████████▋ | 27399/45917 [10:25:54<5:37:50, 1.09s/it][2025-04-24 03:48:05] (step=0027400) Train Loss: 6.3345, Train Steps/Sec: 0.90 + 60%|██████████████████████████████████████████████████████████████▋ | 27424/45917 [10:26:22<5:48:29, 1.13s/it][2025-04-24 03:48:33] (step=0027425) Train Loss: 6.3656, Train Steps/Sec: 0.90 + 60%|██████████████████████████████████████████████████████████████▊ | 27449/45917 [10:26:49<5:39:58, 1.10s/it][2025-04-24 03:49:01] (step=0027450) Train Loss: 6.3844, Train Steps/Sec: 0.91 + 60%|██████████████████████████████████████████████████████████████▊ | 27474/45917 [10:27:17<5:37:50, 1.10s/it][2025-04-24 03:49:28] (step=0027475) Train Loss: 6.4334, Train Steps/Sec: 0.90 + 60%|██████████████████████████████████████████████████████████████▉ | 27499/45917 [10:27:45<5:36:03, 1.09s/it][2025-04-24 03:49:56] (step=0027500) Train Loss: 6.3806, Train Steps/Sec: 0.90 + 60%|██████████████████████████████████████████████████████████████▉ | 27524/45917 [10:28:13<5:48:06, 1.14s/it][2025-04-24 03:50:24] (step=0027525) Train Loss: 6.4157, Train Steps/Sec: 0.90 + 60%|██████████████████████████████████████████████████████████████▉ | 27549/45917 [10:28:41<5:42:11, 1.12s/it][2025-04-24 03:50:52] (step=0027550) Train Loss: 6.3985, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████ | 27574/45917 [10:29:08<5:36:17, 1.10s/it][2025-04-24 03:51:19] (step=0027575) Train Loss: 6.4154, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████ | 27599/45917 [10:29:36<5:32:29, 1.09s/it][2025-04-24 03:51:47] (step=0027600) Train Loss: 6.3770, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████▏ | 27624/45917 [10:30:04<5:45:53, 1.13s/it][2025-04-24 03:52:15] (step=0027625) Train Loss: 6.3465, Train Steps/Sec: 0.91 + 60%|███████████████████████████████████████████████████████████████▏ | 27649/45917 [10:30:31<5:40:21, 1.12s/it][2025-04-24 03:52:43] (step=0027650) Train Loss: 6.3677, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████▎ | 27674/45917 [10:30:59<5:34:08, 1.10s/it][2025-04-24 03:53:10] (step=0027675) Train Loss: 6.3647, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████▎ | 27699/45917 [10:31:27<5:30:50, 1.09s/it][2025-04-24 03:53:38] (step=0027700) Train Loss: 6.3514, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████▍ | 27724/45917 [10:31:54<5:42:15, 1.13s/it][2025-04-24 03:54:06] (step=0027725) Train Loss: 6.3652, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████▍ | 27749/45917 [10:32:22<5:40:54, 1.13s/it][2025-04-24 03:54:34] (step=0027750) Train Loss: 6.4131, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████▌ | 27774/45917 [10:32:50<5:32:07, 1.10s/it][2025-04-24 03:55:01] (step=0027775) Train Loss: 6.3351, Train Steps/Sec: 0.91 + 61%|███████████████████████████████████████████████████████████████▌ | 27799/45917 [10:33:18<5:30:12, 1.09s/it][2025-04-24 03:55:29] (step=0027800) Train Loss: 6.3897, Train Steps/Sec: 0.90 + 61%|███████████████████████████████████████████████████████████████▋ | 27824/45917 [10:33:45<5:39:24, 1.13s/it][2025-04-24 03:55:57] (step=0027825) Train Loss: 6.3987, Train Steps/Sec: 0.90 + 61%|███████████████████████████████████████████████████████████████▋ | 27849/45917 [10:34:13<5:31:53, 1.10s/it][2025-04-24 03:56:24] (step=0027850) Train Loss: 6.3973, Train Steps/Sec: 0.90 + 61%|███████████████████████████████████████████████████████████████▋ | 27874/45917 [10:34:41<5:27:42, 1.09s/it][2025-04-24 03:56:52] (step=0027875) Train Loss: 6.3826, Train Steps/Sec: 0.91 + 61%|███████████████████████████████████████████████████████████████▊ | 27899/45917 [10:35:08<5:27:29, 1.09s/it][2025-04-24 03:57:20] (step=0027900) Train Loss: 6.3941, Train Steps/Sec: 0.90 + 61%|███████████████████████████████████████████████████████████████▊ | 27924/45917 [10:35:36<5:35:40, 1.12s/it][2025-04-24 03:57:48] (step=0027925) Train Loss: 6.3666, Train Steps/Sec: 0.90 + 61%|███████████████████████████████████████████████████████████████▉ | 27949/45917 [10:36:04<5:31:56, 1.11s/it][2025-04-24 03:58:15] (step=0027950) Train Loss: 6.3905, Train Steps/Sec: 0.90 + 61%|███████████████████████████████████████████████████████████████▉ | 27974/45917 [10:36:32<5:31:15, 1.11s/it][2025-04-24 03:58:43] (step=0027975) Train Loss: 6.3974, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████ | 27999/45917 [10:36:59<5:28:50, 1.10s/it][2025-04-24 03:59:11] (step=0028000) Train Loss: 6.3700, Train Steps/Sec: 0.90 +[2025-04-24 03:59:11] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 03:59:11] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 03:59:11] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.40s/it] +[2025-04-24 04:03:54] Finish Eval in 28000 steps...█████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.03s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 04:04:26] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0028000.pt +[2025-04-24 04:04:26] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0026000.pt + 61%|████████████████████████████████████████████████████████████████ | 28024/45917 [10:42:42<5:42:04, 1.15s/it][2025-04-24 04:04:54] (step=0028025) Train Loss: 6.3272, Train Steps/Sec: 0.07 + 61%|████████████████████████████████████████████████████████████████▏ | 28049/45917 [10:43:10<5:31:22, 1.11s/it][2025-04-24 04:05:21] (step=0028050) Train Loss: 6.4020, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████▏ | 28074/45917 [10:43:38<5:26:32, 1.10s/it][2025-04-24 04:05:49] (step=0028075) Train Loss: 6.3601, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████▎ | 28099/45917 [10:44:06<5:24:25, 1.09s/it][2025-04-24 04:06:17] (step=0028100) Train Loss: 6.3739, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████▎ | 28124/45917 [10:44:33<5:35:28, 1.13s/it][2025-04-24 04:06:45] (step=0028125) Train Loss: 6.3575, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████▎ | 28149/45917 [10:45:01<5:28:50, 1.11s/it][2025-04-24 04:07:12] (step=0028150) Train Loss: 6.3462, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████▍ | 28174/45917 [10:45:29<5:24:48, 1.10s/it][2025-04-24 04:07:40] (step=0028175) Train Loss: 6.3810, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████▍ | 28199/45917 [10:45:57<5:23:22, 1.10s/it][2025-04-24 04:08:08] (step=0028200) Train Loss: 6.3267, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████▌ | 28224/45917 [10:46:24<5:32:10, 1.13s/it][2025-04-24 04:08:36] (step=0028225) Train Loss: 6.4060, Train Steps/Sec: 0.91 + 62%|████████████████████████████████████████████████████████████████▌ | 28249/45917 [10:46:52<5:25:53, 1.11s/it][2025-04-24 04:09:03] (step=0028250) Train Loss: 6.4051, Train Steps/Sec: 0.90 + 62%|████████████████████████████████████████████████████████████████▋ | 28274/45917 [10:47:20<5:20:56, 1.09s/it][2025-04-24 04:09:31] (step=0028275) Train Loss: 6.3547, Train Steps/Sec: 0.90 + 62%|████████████████████████████████████████████████████████████████▋ | 28299/45917 [10:47:47<5:20:26, 1.09s/it][2025-04-24 04:09:59] (step=0028300) Train Loss: 6.3960, Train Steps/Sec: 0.90 + 62%|████████████████████████████████████████████████████████████████▊ | 28324/45917 [10:48:15<5:31:37, 1.13s/it][2025-04-24 04:10:26] (step=0028325) Train Loss: 6.3940, Train Steps/Sec: 0.91 + 62%|████████████████████████████████████████████████████████████████▊ | 28349/45917 [10:48:43<5:26:46, 1.12s/it][2025-04-24 04:10:54] (step=0028350) Train Loss: 6.3788, Train Steps/Sec: 0.90 + 62%|████████████████████████████████████████████████████████████████▉ | 28374/45917 [10:49:10<5:20:16, 1.10s/it][2025-04-24 04:11:22] (step=0028375) Train Loss: 6.3520, Train Steps/Sec: 0.90 + 62%|████████████████████████████████████████████████████████████████▉ | 28399/45917 [10:49:38<5:17:42, 1.09s/it][2025-04-24 04:11:49] (step=0028400) Train Loss: 6.3477, Train Steps/Sec: 0.90 + 62%|████████████████████████████████████████████████████████████████▉ | 28424/45917 [10:50:06<5:26:33, 1.12s/it][2025-04-24 04:12:17] (step=0028425) Train Loss: 6.3400, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████ | 28449/45917 [10:50:33<5:20:14, 1.10s/it][2025-04-24 04:12:45] (step=0028450) Train Loss: 6.3274, Train Steps/Sec: 0.91 + 62%|█████████████████████████████████████████████████████████████████ | 28474/45917 [10:51:01<5:21:17, 1.11s/it][2025-04-24 04:13:12] (step=0028475) Train Loss: 6.3621, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████▏ | 28499/45917 [10:51:29<5:18:21, 1.10s/it][2025-04-24 04:13:40] (step=0028500) Train Loss: 6.3798, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████▏ | 28524/45917 [10:51:57<5:26:19, 1.13s/it][2025-04-24 04:14:08] (step=0028525) Train Loss: 6.3716, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████▎ | 28549/45917 [10:52:24<5:21:51, 1.11s/it][2025-04-24 04:14:36] (step=0028550) Train Loss: 6.3779, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████▎ | 28574/45917 [10:52:52<5:16:15, 1.09s/it][2025-04-24 04:15:03] (step=0028575) Train Loss: 6.3372, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████▍ | 28599/45917 [10:53:20<5:14:52, 1.09s/it][2025-04-24 04:15:31] (step=0028600) Train Loss: 6.3641, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████▍ | 28624/45917 [10:53:48<5:27:21, 1.14s/it][2025-04-24 04:15:59] (step=0028625) Train Loss: 6.3758, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████▌ | 28649/45917 [10:54:15<5:15:53, 1.10s/it][2025-04-24 04:16:26] (step=0028650) Train Loss: 6.3707, Train Steps/Sec: 0.91 + 62%|█████████████████████████████████████████████████████████████████▌ | 28674/45917 [10:54:43<5:13:50, 1.09s/it][2025-04-24 04:16:54] (step=0028675) Train Loss: 6.3897, Train Steps/Sec: 0.90 + 63%|█████████████████████████████████████████████████████████████████▋ | 28699/45917 [10:55:11<5:16:24, 1.10s/it][2025-04-24 04:17:22] (step=0028700) Train Loss: 6.3760, Train Steps/Sec: 0.89 + 63%|█████████████████████████████████████████████████████████████████▋ | 28724/45917 [10:55:39<5:20:00, 1.12s/it][2025-04-24 04:17:50] (step=0028725) Train Loss: 6.4067, Train Steps/Sec: 0.90 + 63%|█████████████████████████████████████████████████████████████████▋ | 28749/45917 [10:56:06<5:17:12, 1.11s/it][2025-04-24 04:18:18] (step=0028750) Train Loss: 6.3833, Train Steps/Sec: 0.90 + 63%|█████████████████████████████████████████████████████████████████▊ | 28774/45917 [10:56:34<5:17:30, 1.11s/it][2025-04-24 04:18:45] (step=0028775) Train Loss: 6.3495, Train Steps/Sec: 0.90 + 63%|█████████████████████████████████████████████████████████████████▊ | 28799/45917 [10:57:02<5:12:09, 1.09s/it][2025-04-24 04:19:13] (step=0028800) Train Loss: 6.3568, Train Steps/Sec: 0.90 + 63%|█████████████████████████████████████████████████████████████████▉ | 28824/45917 [10:57:29<5:17:57, 1.12s/it][2025-04-24 04:19:41] (step=0028825) Train Loss: 6.3480, Train Steps/Sec: 0.90 + 63%|█████████████████████████████████████████████████████████████████▉ | 28849/45917 [10:57:57<5:16:41, 1.11s/it][2025-04-24 04:20:09] (step=0028850) Train Loss: 6.3906, Train Steps/Sec: 0.90 + 63%|██████████████████████████████████████████████████████████████████ | 28874/45917 [10:58:25<5:16:08, 1.11s/it][2025-04-24 04:20:36] (step=0028875) Train Loss: 6.3455, Train Steps/Sec: 0.90 + 63%|██████████████████████████████████████████████████████████████████ | 28899/45917 [10:58:53<5:14:55, 1.11s/it][2025-04-24 04:21:04] (step=0028900) Train Loss: 6.3649, Train Steps/Sec: 0.90 + 63%|██████████████████████████████████████████████████████████████████▏ | 28924/45917 [10:59:21<5:17:28, 1.12s/it][2025-04-24 04:21:32] (step=0028925) Train Loss: 6.3775, Train Steps/Sec: 0.91 + 63%|██████████████████████████████████████████████████████████████████▏ | 28949/45917 [10:59:48<5:13:28, 1.11s/it][2025-04-24 04:22:00] (step=0028950) Train Loss: 6.3932, Train Steps/Sec: 0.90 + 63%|██████████████████████████████████████████████████████████████████▎ | 28974/45917 [11:00:16<5:13:06, 1.11s/it][2025-04-24 04:22:28] (step=0028975) Train Loss: 6.3752, Train Steps/Sec: 0.89 + 63%|██████████████████████████████████████████████████████████████████▎ | 28999/45917 [11:00:44<5:09:25, 1.10s/it][2025-04-24 04:22:56] (step=0029000) Train Loss: 6.4085, Train Steps/Sec: 0.90 + 63%|██████████████████████████████████████████████████████████████████▎ | 29024/45917 [11:01:12<5:19:23, 1.13s/it][2025-04-24 04:23:23] (step=0029025) Train Loss: 6.3958, Train Steps/Sec: 0.90 + 63%|██████████████████████████████████████████████████████████████████▍ | 29049/45917 [11:01:40<5:10:04, 1.10s/it][2025-04-24 04:23:51] (step=0029050) Train Loss: 6.3686, Train Steps/Sec: 0.90 + 63%|██████████████████████████████████████████████████████████████████▍ | 29074/45917 [11:02:07<5:06:46, 1.09s/it][2025-04-24 04:24:19] (step=0029075) Train Loss: 6.4163, Train Steps/Sec: 0.91 + 63%|██████████████████████████████████████████████████████████████████▌ | 29099/45917 [11:02:35<5:08:03, 1.10s/it][2025-04-24 04:24:46] (step=0029100) Train Loss: 6.4032, Train Steps/Sec: 0.90 + 63%|██████████████████████████████████████████████████████████████████▌ | 29124/45917 [11:03:03<5:13:45, 1.12s/it][2025-04-24 04:25:14] (step=0029125) Train Loss: 6.3410, Train Steps/Sec: 0.91 + 63%|██████████████████████████████████████████████████████████████████▋ | 29149/45917 [11:03:30<5:10:52, 1.11s/it][2025-04-24 04:25:42] (step=0029150) Train Loss: 6.3686, Train Steps/Sec: 0.90 + 64%|██████████████████████████████████████████████████████████████████▋ | 29174/45917 [11:03:58<5:05:56, 1.10s/it][2025-04-24 04:26:09] (step=0029175) Train Loss: 6.3802, Train Steps/Sec: 0.90 + 64%|██████████████████████████████████████████████████████████████████▊ | 29199/45917 [11:04:26<5:06:12, 1.10s/it][2025-04-24 04:26:37] (step=0029200) Train Loss: 6.4322, Train Steps/Sec: 0.90 + 64%|██████████████████████████████████████████████████████████████████▊ | 29224/45917 [11:04:54<5:15:13, 1.13s/it][2025-04-24 04:27:05] (step=0029225) Train Loss: 6.4052, Train Steps/Sec: 0.90 + 64%|██████████████████████████████████████████████████████████████████▉ | 29249/45917 [11:05:21<5:08:16, 1.11s/it][2025-04-24 04:27:33] (step=0029250) Train Loss: 6.3550, Train Steps/Sec: 0.90 + 64%|██████████████████████████████████████████████████████████████████▉ | 29274/45917 [11:05:49<5:05:59, 1.10s/it][2025-04-24 04:28:00] (step=0029275) Train Loss: 6.4389, Train Steps/Sec: 0.90 + 64%|██████████████████████████████████████████████████████████████████▉ | 29299/45917 [11:06:17<5:01:44, 1.09s/it][2025-04-24 04:28:28] (step=0029300) Train Loss: 6.3862, Train Steps/Sec: 0.90 + 64%|███████████████████████████████████████████████████████████████████ | 29324/45917 [11:06:44<5:11:28, 1.13s/it][2025-04-24 04:28:56] (step=0029325) Train Loss: 6.3689, Train Steps/Sec: 0.90 + 64%|███████████████████████████████████████████████████████████████████ | 29349/45917 [11:07:12<5:05:38, 1.11s/it][2025-04-24 04:29:23] (step=0029350) Train Loss: 6.4123, Train Steps/Sec: 0.91 + 64%|███████████████████████████████████████████████████████████████████▏ | 29374/45917 [11:07:40<5:04:18, 1.10s/it][2025-04-24 04:29:51] (step=0029375) Train Loss: 6.3432, Train Steps/Sec: 0.90 + 64%|███████████████████████████████████████████████████████████████████▏ | 29399/45917 [11:08:07<4:59:01, 1.09s/it][2025-04-24 04:30:19] (step=0029400) Train Loss: 6.3980, Train Steps/Sec: 0.90 + 64%|███████████████████████████████████████████████████████████████████▎ | 29424/45917 [11:08:35<5:07:23, 1.12s/it][2025-04-24 04:30:46] (step=0029425) Train Loss: 6.3847, Train Steps/Sec: 0.91 + 64%|███████████████████████████████████████████████████████████████████▎ | 29449/45917 [11:09:03<5:06:40, 1.12s/it][2025-04-24 04:31:14] (step=0029450) Train Loss: 6.3612, Train Steps/Sec: 0.90 + 64%|███████████████████████████████████████████████████████████████████▍ | 29474/45917 [11:09:31<5:01:29, 1.10s/it][2025-04-24 04:31:42] (step=0029475) Train Loss: 6.3767, Train Steps/Sec: 0.90 + 64%|███████████████████████████████████████████████████████████████████▍ | 29499/45917 [11:09:58<5:03:21, 1.11s/it][2025-04-24 04:32:10] (step=0029500) Train Loss: 6.3897, Train Steps/Sec: 0.89 + 64%|███████████████████████████████████████████████████████████████████▌ | 29524/45917 [11:10:26<5:10:06, 1.14s/it][2025-04-24 04:32:38] (step=0029525) Train Loss: 6.3857, Train Steps/Sec: 0.90 + 64%|███████████████████████████████████████████████████████████████████▌ | 29549/45917 [11:10:54<5:01:40, 1.11s/it][2025-04-24 04:33:05] (step=0029550) Train Loss: 6.3779, Train Steps/Sec: 0.90 + 64%|███████████████████████████████████████████████████████████████████▋ | 29574/45917 [11:11:22<5:01:27, 1.11s/it][2025-04-24 04:33:33] (step=0029575) Train Loss: 6.3194, Train Steps/Sec: 0.90 + 64%|███████████████████████████████████████████████████████████████████▋ | 29599/45917 [11:11:50<5:03:59, 1.12s/it][2025-04-24 04:34:01] (step=0029600) Train Loss: 6.3982, Train Steps/Sec: 0.89 + 65%|███████████████████████████████████████████████████████████████████▋ | 29624/45917 [11:12:17<5:06:19, 1.13s/it][2025-04-24 04:34:29] (step=0029625) Train Loss: 6.3716, Train Steps/Sec: 0.90 + 65%|███████████████████████████████████████████████████████████████████▊ | 29649/45917 [11:12:45<5:01:57, 1.11s/it][2025-04-24 04:34:57] (step=0029650) Train Loss: 6.3872, Train Steps/Sec: 0.90 + 65%|███████████████████████████████████████████████████████████████████▊ | 29674/45917 [11:13:13<4:58:54, 1.10s/it][2025-04-24 04:35:24] (step=0029675) Train Loss: 6.3961, Train Steps/Sec: 0.90 + 65%|███████████████████████████████████████████████████████████████████▉ | 29699/45917 [11:13:41<4:55:38, 1.09s/it][2025-04-24 04:35:52] (step=0029700) Train Loss: 6.3724, Train Steps/Sec: 0.89 + 65%|███████████████████████████████████████████████████████████████████▉ | 29724/45917 [11:14:09<5:03:28, 1.12s/it][2025-04-24 04:36:20] (step=0029725) Train Loss: 6.4301, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████ | 29749/45917 [11:14:36<5:03:01, 1.12s/it][2025-04-24 04:36:48] (step=0029750) Train Loss: 6.3691, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████ | 29774/45917 [11:15:04<4:56:46, 1.10s/it][2025-04-24 04:37:15] (step=0029775) Train Loss: 6.3392, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████▏ | 29799/45917 [11:15:32<4:53:53, 1.09s/it][2025-04-24 04:37:43] (step=0029800) Train Loss: 6.3862, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████▏ | 29824/45917 [11:16:00<5:02:17, 1.13s/it][2025-04-24 04:38:11] (step=0029825) Train Loss: 6.4247, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████▎ | 29849/45917 [11:16:27<4:59:09, 1.12s/it][2025-04-24 04:38:39] (step=0029850) Train Loss: 6.3645, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████▎ | 29874/45917 [11:16:55<4:54:59, 1.10s/it][2025-04-24 04:39:06] (step=0029875) Train Loss: 6.3501, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████▎ | 29899/45917 [11:17:23<4:52:28, 1.10s/it][2025-04-24 04:39:34] (step=0029900) Train Loss: 6.3325, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████▍ | 29924/45917 [11:17:51<5:00:14, 1.13s/it][2025-04-24 04:40:02] (step=0029925) Train Loss: 6.3367, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████▍ | 29949/45917 [11:18:18<4:57:20, 1.12s/it][2025-04-24 04:40:29] (step=0029950) Train Loss: 6.3584, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████▌ | 29974/45917 [11:18:46<4:56:04, 1.11s/it][2025-04-24 04:40:57] (step=0029975) Train Loss: 6.2757, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████▌ | 29999/45917 [11:19:14<4:52:05, 1.10s/it][2025-04-24 04:41:25] (step=0030000) Train Loss: 6.3701, Train Steps/Sec: 0.90 +[2025-04-24 04:41:25] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 04:41:25] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 04:41:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.60s/it] +[2025-04-24 04:46:11] Finish Eval in 30000 steps...█████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.16s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 04:46:33] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0030000.pt +[2025-04-24 04:46:35] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0028000.pt + 65%|████████████████████████████████████████████████████████████████████▋ | 30024/45917 [11:24:52<5:03:44, 1.15s/it][2025-04-24 04:47:03] (step=0030025) Train Loss: 6.3995, Train Steps/Sec: 0.07 + 65%|████████████████████████████████████████████████████████████████████▋ | 30049/45917 [11:25:20<4:53:54, 1.11s/it][2025-04-24 04:47:31] (step=0030050) Train Loss: 6.3457, Train Steps/Sec: 0.90 + 65%|████████████████████████████████████████████████████████████████████▊ | 30074/45917 [11:25:48<4:49:25, 1.10s/it][2025-04-24 04:47:59] (step=0030075) Train Loss: 6.4053, Train Steps/Sec: 0.90 + 66%|████████████████████████████████████████████████████████████████████▊ | 30099/45917 [11:26:15<4:51:44, 1.11s/it][2025-04-24 04:48:27] (step=0030100) Train Loss: 6.3528, Train Steps/Sec: 0.90 + 66%|████████████████████████████████████████████████████████████████████▉ | 30124/45917 [11:26:43<4:55:04, 1.12s/it][2025-04-24 04:48:54] (step=0030125) Train Loss: 6.3571, Train Steps/Sec: 0.90 + 66%|████████████████████████████████████████████████████████████████████▉ | 30149/45917 [11:27:11<4:53:22, 1.12s/it][2025-04-24 04:49:22] (step=0030150) Train Loss: 6.4028, Train Steps/Sec: 0.90 + 66%|████████████████████████████████████████████████████████████████████▉ | 30174/45917 [11:27:39<4:50:31, 1.11s/it][2025-04-24 04:49:50] (step=0030175) Train Loss: 6.4188, Train Steps/Sec: 0.90 + 66%|█████████████████████████████████████████████████████████████████████ | 30199/45917 [11:28:07<4:47:45, 1.10s/it][2025-04-24 04:50:18] (step=0030200) Train Loss: 6.4204, Train Steps/Sec: 0.89 + 66%|█████████████████████████████████████████████████████████████████████ | 30224/45917 [11:28:34<4:56:29, 1.13s/it][2025-04-24 04:50:46] (step=0030225) Train Loss: 6.3821, Train Steps/Sec: 0.90 + 66%|█████████████████████████████████████████████████████████████████████▏ | 30249/45917 [11:29:02<4:50:00, 1.11s/it][2025-04-24 04:51:13] (step=0030250) Train Loss: 6.3786, Train Steps/Sec: 0.90 + 66%|█████████████████████████████████████████████████████████████████████▏ | 30274/45917 [11:29:30<4:47:33, 1.10s/it][2025-04-24 04:51:41] (step=0030275) Train Loss: 6.3792, Train Steps/Sec: 0.90 + 66%|█████████████████████████████████████████████████████████████████████▎ | 30299/45917 [11:29:58<4:46:15, 1.10s/it][2025-04-24 04:52:09] (step=0030300) Train Loss: 6.4309, Train Steps/Sec: 0.90 + 66%|█████████████████████████████████████████████████████████████████████▎ | 30324/45917 [11:30:26<4:51:43, 1.12s/it][2025-04-24 04:52:37] (step=0030325) Train Loss: 6.3569, Train Steps/Sec: 0.90 + 66%|█████████████████████████████████████████████████████████████████████▍ | 30349/45917 [11:30:53<4:47:07, 1.11s/it][2025-04-24 04:53:04] (step=0030350) Train Loss: 6.4439, Train Steps/Sec: 0.91 + 66%|█████████████████████████████████████████████████████████████████████▍ | 30374/45917 [11:31:21<4:45:04, 1.10s/it][2025-04-24 04:53:32] (step=0030375) Train Loss: 6.3710, Train Steps/Sec: 0.90 + 66%|█████████████████████████████████████████████████████████████████████▌ | 30399/45917 [11:31:49<4:44:52, 1.10s/it][2025-04-24 04:54:00] (step=0030400) Train Loss: 6.3640, Train Steps/Sec: 0.90 + 66%|█████████████████████████████████████████████████████████████████████▌ | 30424/45917 [11:32:28<8:10:37, 1.90s/it][2025-04-24 04:54:39] (step=0030425) Train Loss: 6.3840, Train Steps/Sec: 0.63 + 66%|█████████████████████████████████████████████████████████████████████▋ | 30449/45917 [11:32:56<4:46:58, 1.11s/it][2025-04-24 04:55:07] (step=0030450) Train Loss: 6.3794, Train Steps/Sec: 0.90 + 66%|█████████████████████████████████████████████████████████████████████▋ | 30474/45917 [11:33:24<4:43:01, 1.10s/it][2025-04-24 04:55:35] (step=0030475) Train Loss: 6.3663, Train Steps/Sec: 0.90 + 66%|█████████████████████████████████████████████████████████████████████▋ | 30499/45917 [11:33:51<4:42:58, 1.10s/it][2025-04-24 04:56:03] (step=0030500) Train Loss: 6.3721, Train Steps/Sec: 0.90 + 66%|█████████████████████████████████████████████████████████████████████▊ | 30524/45917 [11:34:20<5:20:39, 1.25s/it][2025-04-24 04:56:31] (step=0030525) Train Loss: 6.3799, Train Steps/Sec: 0.89 + 67%|█████████████████████████████████████████████████████████████████████▊ | 30549/45917 [11:34:47<4:43:47, 1.11s/it][2025-04-24 04:56:59] (step=0030550) Train Loss: 6.4199, Train Steps/Sec: 0.90 + 67%|█████████████████████████████████████████████████████████████████████▉ | 30574/45917 [11:35:15<4:42:53, 1.11s/it][2025-04-24 04:57:26] (step=0030575) Train Loss: 6.3612, Train Steps/Sec: 0.90 + 67%|█████████████████████████████████████████████████████████████████████▉ | 30599/45917 [11:35:43<4:39:39, 1.10s/it][2025-04-24 04:57:54] (step=0030600) Train Loss: 6.4267, Train Steps/Sec: 0.90 + 67%|██████████████████████████████████████████████████████████████████████ | 30624/45917 [11:36:17<5:14:23, 1.23s/it][2025-04-24 04:58:28] (step=0030625) Train Loss: 6.3407, Train Steps/Sec: 0.74 + 67%|██████████████████████████████████████████████████████████████████████ | 30649/45917 [11:36:44<4:41:46, 1.11s/it][2025-04-24 04:58:56] (step=0030650) Train Loss: 6.4299, Train Steps/Sec: 0.90 + 67%|██████████████████████████████████████████████████████████████████████▏ | 30674/45917 [11:37:12<4:40:46, 1.11s/it][2025-04-24 04:59:23] (step=0030675) Train Loss: 6.3953, Train Steps/Sec: 0.90 + 67%|█████████████████████████████████████████████████████████████████████▌ | 30699/45917 [11:37:52<12:21:41, 2.92s/it][2025-04-24 05:00:03] (step=0030700) Train Loss: 6.3516, Train Steps/Sec: 0.62 + 67%|██████████████████████████████████████████████████████████████████████▎ | 30724/45917 [11:38:20<4:43:47, 1.12s/it][2025-04-24 05:00:31] (step=0030725) Train Loss: 6.4050, Train Steps/Sec: 0.90 + 67%|██████████████████████████████████████████████████████████████████████▎ | 30749/45917 [11:38:47<4:40:30, 1.11s/it][2025-04-24 05:00:59] (step=0030750) Train Loss: 6.3638, Train Steps/Sec: 0.90 + 67%|██████████████████████████████████████████████████████████████████████▎ | 30774/45917 [11:39:15<4:37:24, 1.10s/it][2025-04-24 05:01:26] (step=0030775) Train Loss: 6.3976, Train Steps/Sec: 0.90 + 67%|██████████████████████████████████████████████████████████████████████▍ | 30799/45917 [11:39:43<4:37:41, 1.10s/it][2025-04-24 05:01:54] (step=0030800) Train Loss: 6.4337, Train Steps/Sec: 0.90 + 67%|██████████████████████████████████████████████████████████████████████▍ | 30824/45917 [11:40:11<4:40:19, 1.11s/it][2025-04-24 05:02:22] (step=0030825) Train Loss: 6.3549, Train Steps/Sec: 0.91 + 67%|██████████████████████████████████████████████████████████████████████▌ | 30849/45917 [11:40:38<4:42:02, 1.12s/it][2025-04-24 05:02:50] (step=0030850) Train Loss: 6.3504, Train Steps/Sec: 0.90 + 67%|██████████████████████████████████████████████████████████████████████▌ | 30874/45917 [11:41:06<4:37:45, 1.11s/it][2025-04-24 05:03:17] (step=0030875) Train Loss: 6.3821, Train Steps/Sec: 0.90 + 67%|██████████████████████████████████████████████████████████████████████▋ | 30899/45917 [11:41:34<4:36:54, 1.11s/it][2025-04-24 05:03:45] (step=0030900) Train Loss: 6.3906, Train Steps/Sec: 0.90 + 67%|██████████████████████████████████████████████████████████████████████▋ | 30924/45917 [11:42:02<4:43:07, 1.13s/it][2025-04-24 05:04:13] (step=0030925) Train Loss: 6.3457, Train Steps/Sec: 0.90 + 67%|██████████████████████████████████████████████████████████████████████▊ | 30949/45917 [11:42:29<4:37:55, 1.11s/it][2025-04-24 05:04:41] (step=0030950) Train Loss: 6.3630, Train Steps/Sec: 0.90 + 67%|██████████████████████████████████████████████████████████████████████▊ | 30974/45917 [11:42:57<4:32:27, 1.09s/it][2025-04-24 05:05:08] (step=0030975) Train Loss: 6.3385, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████▉ | 30999/45917 [11:43:25<4:32:20, 1.10s/it][2025-04-24 05:05:36] (step=0031000) Train Loss: 6.3693, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████▉ | 31024/45917 [11:43:52<4:39:37, 1.13s/it][2025-04-24 05:06:04] (step=0031025) Train Loss: 6.3512, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████ | 31049/45917 [11:44:20<4:35:43, 1.11s/it][2025-04-24 05:06:31] (step=0031050) Train Loss: 6.3615, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████ | 31074/45917 [11:44:48<4:35:55, 1.12s/it][2025-04-24 05:06:59] (step=0031075) Train Loss: 6.3680, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████ | 31099/45917 [11:45:16<4:31:42, 1.10s/it][2025-04-24 05:07:27] (step=0031100) Train Loss: 6.3453, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▏ | 31124/45917 [11:45:44<4:36:45, 1.12s/it][2025-04-24 05:07:55] (step=0031125) Train Loss: 6.3861, Train Steps/Sec: 0.91 + 68%|███████████████████████████████████████████████████████████████████████▏ | 31149/45917 [11:46:11<4:33:52, 1.11s/it][2025-04-24 05:08:22] (step=0031150) Train Loss: 6.3365, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▎ | 31174/45917 [11:46:39<4:29:40, 1.10s/it][2025-04-24 05:08:50] (step=0031175) Train Loss: 6.3323, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▎ | 31199/45917 [11:47:07<4:29:31, 1.10s/it][2025-04-24 05:09:18] (step=0031200) Train Loss: 6.3535, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▍ | 31224/45917 [11:47:35<4:36:05, 1.13s/it][2025-04-24 05:09:46] (step=0031225) Train Loss: 6.3467, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▍ | 31249/45917 [11:48:02<4:30:24, 1.11s/it][2025-04-24 05:10:13] (step=0031250) Train Loss: 6.3217, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▌ | 31274/45917 [11:48:30<4:28:13, 1.10s/it][2025-04-24 05:10:41] (step=0031275) Train Loss: 6.2912, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▌ | 31299/45917 [11:48:58<4:26:11, 1.09s/it][2025-04-24 05:11:09] (step=0031300) Train Loss: 6.3135, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▋ | 31324/45917 [11:49:26<4:35:47, 1.13s/it][2025-04-24 05:11:37] (step=0031325) Train Loss: 6.3718, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▋ | 31349/45917 [11:49:53<4:30:28, 1.11s/it][2025-04-24 05:12:05] (step=0031350) Train Loss: 6.3599, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▋ | 31374/45917 [11:50:21<4:27:15, 1.10s/it][2025-04-24 05:12:32] (step=0031375) Train Loss: 6.3857, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▊ | 31399/45917 [11:50:49<4:25:18, 1.10s/it][2025-04-24 05:13:00] (step=0031400) Train Loss: 6.3958, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████▊ | 31424/45917 [11:51:16<4:30:41, 1.12s/it][2025-04-24 05:13:28] (step=0031425) Train Loss: 6.4175, Train Steps/Sec: 0.91 + 68%|███████████████████████████████████████████████████████████████████████▉ | 31449/45917 [11:51:44<4:26:01, 1.10s/it][2025-04-24 05:13:55] (step=0031450) Train Loss: 6.3857, Train Steps/Sec: 0.90 + 69%|███████████████████████████████████████████████████████████████████████▉ | 31474/45917 [11:52:12<4:24:08, 1.10s/it][2025-04-24 05:14:23] (step=0031475) Train Loss: 6.4115, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████ | 31499/45917 [11:52:40<4:26:10, 1.11s/it][2025-04-24 05:14:51] (step=0031500) Train Loss: 6.3964, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████ | 31524/45917 [11:53:07<4:31:55, 1.13s/it][2025-04-24 05:15:19] (step=0031525) Train Loss: 6.3871, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▏ | 31549/45917 [11:53:35<4:26:50, 1.11s/it][2025-04-24 05:15:46] (step=0031550) Train Loss: 6.4048, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▏ | 31574/45917 [11:54:03<4:26:47, 1.12s/it][2025-04-24 05:16:14] (step=0031575) Train Loss: 6.3523, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▎ | 31599/45917 [11:54:31<4:20:14, 1.09s/it][2025-04-24 05:16:42] (step=0031600) Train Loss: 6.3736, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▎ | 31624/45917 [11:54:58<4:28:20, 1.13s/it][2025-04-24 05:17:10] (step=0031625) Train Loss: 6.3597, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▎ | 31649/45917 [11:55:26<4:23:34, 1.11s/it][2025-04-24 05:17:37] (step=0031650) Train Loss: 6.3740, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▍ | 31674/45917 [11:55:54<4:22:36, 1.11s/it][2025-04-24 05:18:05] (step=0031675) Train Loss: 6.3336, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▍ | 31699/45917 [11:56:22<4:19:44, 1.10s/it][2025-04-24 05:18:33] (step=0031700) Train Loss: 6.3346, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▌ | 31724/45917 [11:56:49<4:26:46, 1.13s/it][2025-04-24 05:19:00] (step=0031725) Train Loss: 6.3398, Train Steps/Sec: 0.91 + 69%|████████████████████████████████████████████████████████████████████████▌ | 31749/45917 [11:57:17<4:21:46, 1.11s/it][2025-04-24 05:19:28] (step=0031750) Train Loss: 6.3721, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▋ | 31774/45917 [11:57:45<4:19:51, 1.10s/it][2025-04-24 05:19:56] (step=0031775) Train Loss: 6.4125, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▋ | 31799/45917 [11:58:19<4:18:21, 1.10s/it][2025-04-24 05:20:31] (step=0031800) Train Loss: 6.3726, Train Steps/Sec: 0.72 + 69%|████████████████████████████████████████████████████████████████████████▊ | 31824/45917 [11:58:55<4:31:35, 1.16s/it][2025-04-24 05:21:06] (step=0031825) Train Loss: 6.3837, Train Steps/Sec: 0.70 + 69%|████████████████████████████████████████████████████████████████████████▊ | 31849/45917 [11:59:23<4:20:00, 1.11s/it][2025-04-24 05:21:34] (step=0031850) Train Loss: 6.3031, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▉ | 31874/45917 [11:59:51<4:17:49, 1.10s/it][2025-04-24 05:22:02] (step=0031875) Train Loss: 6.3547, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████▉ | 31899/45917 [12:00:19<4:18:56, 1.11s/it][2025-04-24 05:22:30] (step=0031900) Train Loss: 6.3718, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████ | 31924/45917 [12:00:46<4:23:49, 1.13s/it][2025-04-24 05:22:57] (step=0031925) Train Loss: 6.3584, Train Steps/Sec: 0.91 + 70%|█████████████████████████████████████████████████████████████████████████ | 31949/45917 [12:01:14<4:17:24, 1.11s/it][2025-04-24 05:23:25] (step=0031950) Train Loss: 6.4172, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████ | 31974/45917 [12:01:42<4:16:50, 1.11s/it][2025-04-24 05:23:53] (step=0031975) Train Loss: 6.3192, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████▏ | 31999/45917 [12:02:10<4:14:27, 1.10s/it][2025-04-24 05:24:21] (step=0032000) Train Loss: 6.3708, Train Steps/Sec: 0.90 +[2025-04-24 05:24:21] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 05:24:21] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 05:24:21] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.46s/it] +[2025-04-24 05:29:06] Finish Eval in 32000 steps...█████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.09s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 05:29:30] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0032000.pt +[2025-04-24 05:29:32] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0030000.pt + 70%|█████████████████████████████████████████████████████████████████████████▏ | 32024/45917 [12:07:49<4:26:20, 1.15s/it][2025-04-24 05:30:00] (step=0032025) Train Loss: 6.3625, Train Steps/Sec: 0.07 + 70%|█████████████████████████████████████████████████████████████████████████▎ | 32049/45917 [12:08:17<4:16:29, 1.11s/it][2025-04-24 05:30:28] (step=0032050) Train Loss: 6.4276, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████▎ | 32074/45917 [12:08:44<4:12:16, 1.09s/it][2025-04-24 05:30:56] (step=0032075) Train Loss: 6.3588, Train Steps/Sec: 0.91 + 70%|█████████████████████████████████████████████████████████████████████████▍ | 32099/45917 [12:09:12<4:12:31, 1.10s/it][2025-04-24 05:31:23] (step=0032100) Train Loss: 6.3700, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████▍ | 32124/45917 [12:09:40<4:19:13, 1.13s/it][2025-04-24 05:31:51] (step=0032125) Train Loss: 6.3653, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████▌ | 32149/45917 [12:10:08<4:15:41, 1.11s/it][2025-04-24 05:32:19] (step=0032150) Train Loss: 6.3836, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████▌ | 32174/45917 [12:10:35<4:14:02, 1.11s/it][2025-04-24 05:32:47] (step=0032175) Train Loss: 6.4445, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████▋ | 32199/45917 [12:11:03<4:13:01, 1.11s/it][2025-04-24 05:33:14] (step=0032200) Train Loss: 6.3700, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████▋ | 32224/45917 [12:11:31<4:15:42, 1.12s/it][2025-04-24 05:33:42] (step=0032225) Train Loss: 6.3523, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████▋ | 32249/45917 [12:11:59<4:13:35, 1.11s/it][2025-04-24 05:34:10] (step=0032250) Train Loss: 6.3571, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████▊ | 32274/45917 [12:12:26<4:08:28, 1.09s/it][2025-04-24 05:34:37] (step=0032275) Train Loss: 6.3639, Train Steps/Sec: 0.91 + 70%|█████████████████████████████████████████████████████████████████████████▊ | 32299/45917 [12:12:54<4:07:36, 1.09s/it][2025-04-24 05:35:05] (step=0032300) Train Loss: 6.3812, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████▉ | 32324/45917 [12:13:22<4:14:37, 1.12s/it][2025-04-24 05:35:33] (step=0032325) Train Loss: 6.3646, Train Steps/Sec: 0.91 + 70%|█████████████████████████████████████████████████████████████████████████▉ | 32349/45917 [12:13:49<4:12:11, 1.12s/it][2025-04-24 05:36:00] (step=0032350) Train Loss: 6.3425, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████ | 32374/45917 [12:14:17<4:09:03, 1.10s/it][2025-04-24 05:36:28] (step=0032375) Train Loss: 6.3891, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████ | 32399/45917 [12:14:45<4:08:44, 1.10s/it][2025-04-24 05:36:56] (step=0032400) Train Loss: 6.3537, Train Steps/Sec: 0.89 + 71%|██████████████████████████████████████████████████████████████████████████▏ | 32424/45917 [12:15:13<4:13:30, 1.13s/it][2025-04-24 05:37:24] (step=0032425) Train Loss: 6.3954, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▏ | 32449/45917 [12:15:40<4:08:46, 1.11s/it][2025-04-24 05:37:52] (step=0032450) Train Loss: 6.3562, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▎ | 32474/45917 [12:16:08<4:08:56, 1.11s/it][2025-04-24 05:38:19] (step=0032475) Train Loss: 6.3666, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▎ | 32499/45917 [12:16:36<4:04:48, 1.09s/it][2025-04-24 05:38:47] (step=0032500) Train Loss: 6.3972, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▎ | 32524/45917 [12:17:04<4:11:29, 1.13s/it][2025-04-24 05:39:15] (step=0032525) Train Loss: 6.4008, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▍ | 32549/45917 [12:17:31<4:06:36, 1.11s/it][2025-04-24 05:39:43] (step=0032550) Train Loss: 6.3447, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▍ | 32574/45917 [12:17:59<4:04:50, 1.10s/it][2025-04-24 05:40:10] (step=0032575) Train Loss: 6.3544, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▌ | 32599/45917 [12:18:27<4:02:40, 1.09s/it][2025-04-24 05:40:38] (step=0032600) Train Loss: 6.3441, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▌ | 32624/45917 [12:18:55<4:08:32, 1.12s/it][2025-04-24 05:41:06] (step=0032625) Train Loss: 6.3587, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▋ | 32649/45917 [12:19:22<4:02:34, 1.10s/it][2025-04-24 05:41:33] (step=0032650) Train Loss: 6.3767, Train Steps/Sec: 0.91 + 71%|██████████████████████████████████████████████████████████████████████████▋ | 32674/45917 [12:19:50<4:03:47, 1.10s/it][2025-04-24 05:42:01] (step=0032675) Train Loss: 6.3655, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▊ | 32699/45917 [12:20:18<4:01:52, 1.10s/it][2025-04-24 05:42:29] (step=0032700) Train Loss: 6.3829, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▊ | 32724/45917 [12:20:45<4:06:44, 1.12s/it][2025-04-24 05:42:57] (step=0032725) Train Loss: 6.4008, Train Steps/Sec: 0.91 + 71%|██████████████████████████████████████████████████████████████████████████▉ | 32749/45917 [12:21:13<4:01:42, 1.10s/it][2025-04-24 05:43:24] (step=0032750) Train Loss: 6.3472, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████▉ | 32774/45917 [12:21:41<4:00:19, 1.10s/it][2025-04-24 05:43:52] (step=0032775) Train Loss: 6.3458, Train Steps/Sec: 0.90 + 71%|███████████████████████████████████████████████████████████████████████████ | 32799/45917 [12:22:08<3:59:21, 1.09s/it][2025-04-24 05:44:20] (step=0032800) Train Loss: 6.3395, Train Steps/Sec: 0.90 + 71%|███████████████████████████████████████████████████████████████████████████ | 32824/45917 [12:22:36<4:06:14, 1.13s/it][2025-04-24 05:44:47] (step=0032825) Train Loss: 6.3651, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████ | 32849/45917 [12:23:04<4:00:18, 1.10s/it][2025-04-24 05:45:15] (step=0032850) Train Loss: 6.3681, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▏ | 32874/45917 [12:23:32<4:01:00, 1.11s/it][2025-04-24 05:45:43] (step=0032875) Train Loss: 6.3589, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▏ | 32899/45917 [12:23:59<3:56:03, 1.09s/it][2025-04-24 05:46:11] (step=0032900) Train Loss: 6.4016, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▎ | 32924/45917 [12:24:27<4:03:47, 1.13s/it][2025-04-24 05:46:38] (step=0032925) Train Loss: 6.3985, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▎ | 32949/45917 [12:24:55<3:59:57, 1.11s/it][2025-04-24 05:47:06] (step=0032950) Train Loss: 6.3556, Train Steps/Sec: 0.91 + 72%|███████████████████████████████████████████████████████████████████████████▍ | 32974/45917 [12:25:22<3:56:35, 1.10s/it][2025-04-24 05:47:34] (step=0032975) Train Loss: 6.3785, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▍ | 32999/45917 [12:25:50<3:56:24, 1.10s/it][2025-04-24 05:48:01] (step=0033000) Train Loss: 6.3382, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▌ | 33024/45917 [12:26:18<4:02:33, 1.13s/it][2025-04-24 05:48:29] (step=0033025) Train Loss: 6.3462, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▌ | 33049/45917 [12:26:45<3:58:07, 1.11s/it][2025-04-24 05:48:57] (step=0033050) Train Loss: 6.3459, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▋ | 33074/45917 [12:27:13<3:59:01, 1.12s/it][2025-04-24 05:49:25] (step=0033075) Train Loss: 6.3711, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▋ | 33099/45917 [12:27:41<3:56:52, 1.11s/it][2025-04-24 05:49:52] (step=0033100) Train Loss: 6.3738, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▋ | 33124/45917 [12:28:09<4:01:27, 1.13s/it][2025-04-24 05:50:20] (step=0033125) Train Loss: 6.3490, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▊ | 33149/45917 [12:28:37<3:56:30, 1.11s/it][2025-04-24 05:50:48] (step=0033150) Train Loss: 6.3406, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▊ | 33174/45917 [12:29:04<3:56:47, 1.11s/it][2025-04-24 05:51:16] (step=0033175) Train Loss: 6.3971, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████▉ | 33199/45917 [12:29:32<3:53:49, 1.10s/it][2025-04-24 05:51:44] (step=0033200) Train Loss: 6.3379, Train Steps/Sec: 0.89 + 72%|███████████████████████████████████████████████████████████████████████████▉ | 33224/45917 [12:30:00<3:57:28, 1.12s/it][2025-04-24 05:52:11] (step=0033225) Train Loss: 6.3747, Train Steps/Sec: 0.90 + 72%|████████████████████████████████████████████████████████████████████████████ | 33249/45917 [12:30:28<3:54:55, 1.11s/it][2025-04-24 05:52:39] (step=0033250) Train Loss: 6.3852, Train Steps/Sec: 0.90 + 72%|████████████████████████████████████████████████████████████████████████████ | 33274/45917 [12:30:56<3:51:43, 1.10s/it][2025-04-24 05:53:07] (step=0033275) Train Loss: 6.2946, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▏ | 33299/45917 [12:31:23<3:51:38, 1.10s/it][2025-04-24 05:53:35] (step=0033300) Train Loss: 6.3362, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▏ | 33324/45917 [12:31:51<3:56:35, 1.13s/it][2025-04-24 05:54:02] (step=0033325) Train Loss: 6.3705, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▎ | 33349/45917 [12:32:19<3:52:37, 1.11s/it][2025-04-24 05:54:30] (step=0033350) Train Loss: 6.3743, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▎ | 33374/45917 [12:32:47<3:51:44, 1.11s/it][2025-04-24 05:54:58] (step=0033375) Train Loss: 6.3903, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▎ | 33399/45917 [12:33:15<3:48:44, 1.10s/it][2025-04-24 05:55:26] (step=0033400) Train Loss: 6.3152, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▍ | 33424/45917 [12:33:42<3:55:09, 1.13s/it][2025-04-24 05:55:54] (step=0033425) Train Loss: 6.3806, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▍ | 33449/45917 [12:34:10<3:52:08, 1.12s/it][2025-04-24 05:56:21] (step=0033450) Train Loss: 6.3513, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▌ | 33474/45917 [12:34:38<3:48:29, 1.10s/it][2025-04-24 05:56:49] (step=0033475) Train Loss: 6.3153, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▌ | 33499/45917 [12:35:06<3:48:20, 1.10s/it][2025-04-24 05:57:17] (step=0033500) Train Loss: 6.3880, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▋ | 33524/45917 [12:35:33<3:53:15, 1.13s/it][2025-04-24 05:57:44] (step=0033525) Train Loss: 6.4334, Train Steps/Sec: 0.91 + 73%|████████████████████████████████████████████████████████████████████████████▋ | 33549/45917 [12:36:01<3:47:38, 1.10s/it][2025-04-24 05:58:12] (step=0033550) Train Loss: 6.3535, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▊ | 33574/45917 [12:36:29<3:45:50, 1.10s/it][2025-04-24 05:58:40] (step=0033575) Train Loss: 6.3995, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▊ | 33599/45917 [12:36:56<3:44:28, 1.09s/it][2025-04-24 05:59:08] (step=0033600) Train Loss: 6.3617, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▉ | 33624/45917 [12:37:24<3:51:30, 1.13s/it][2025-04-24 05:59:35] (step=0033625) Train Loss: 6.3567, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████▉ | 33649/45917 [12:37:52<3:48:42, 1.12s/it][2025-04-24 06:00:03] (step=0033650) Train Loss: 6.3742, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████ | 33674/45917 [12:38:20<3:45:13, 1.10s/it][2025-04-24 06:00:31] (step=0033675) Train Loss: 6.3374, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████ | 33699/45917 [12:38:47<3:41:31, 1.09s/it][2025-04-24 06:00:59] (step=0033700) Train Loss: 6.4073, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████ | 33724/45917 [12:39:15<3:50:30, 1.13s/it][2025-04-24 06:01:26] (step=0033725) Train Loss: 6.3579, Train Steps/Sec: 0.90 + 74%|█████████████████████████████████████████████████████████████████████████████▏ | 33749/45917 [12:39:43<3:44:42, 1.11s/it][2025-04-24 06:01:54] (step=0033750) Train Loss: 6.3872, Train Steps/Sec: 0.90 + 74%|█████████████████████████████████████████████████████████████████████████████▏ | 33774/45917 [12:40:11<3:44:40, 1.11s/it][2025-04-24 06:02:22] (step=0033775) Train Loss: 6.3930, Train Steps/Sec: 0.90 + 74%|█████████████████████████████████████████████████████████████████████████████▎ | 33799/45917 [12:40:38<3:43:26, 1.11s/it][2025-04-24 06:02:50] (step=0033800) Train Loss: 6.3559, Train Steps/Sec: 0.89 + 74%|█████████████████████████████████████████████████████████████████████████████▎ | 33824/45917 [12:41:06<3:47:10, 1.13s/it][2025-04-24 06:03:17] (step=0033825) Train Loss: 6.3379, Train Steps/Sec: 0.90 + 74%|█████████████████████████████████████████████████████████████████████████████▍ | 33849/45917 [12:41:34<3:44:31, 1.12s/it][2025-04-24 06:03:45] (step=0033850) Train Loss: 6.3557, Train Steps/Sec: 0.90 + 74%|█████████████████████████████████████████████████████████████████████████████▍ | 33874/45917 [12:42:02<3:41:09, 1.10s/it][2025-04-24 06:04:13] (step=0033875) Train Loss: 6.3655, Train Steps/Sec: 0.90 + 74%|█████████████████████████████████████████████████████████████████████████████▌ | 33899/45917 [12:42:29<3:39:21, 1.10s/it][2025-04-24 06:04:41] (step=0033900) Train Loss: 6.3296, Train Steps/Sec: 0.90 + 74%|█████████████████████████████████████████████████████████████████████████████▌ | 33924/45917 [12:42:57<3:46:27, 1.13s/it][2025-04-24 06:05:08] (step=0033925) Train Loss: 6.4172, Train Steps/Sec: 0.90 + 74%|█████████████████████████████████████████████████████████████████████████████▋ | 33949/45917 [12:43:25<3:39:15, 1.10s/it][2025-04-24 06:05:36] (step=0033950) Train Loss: 6.3418, Train Steps/Sec: 0.91 + 74%|█████████████████████████████████████████████████████████████████████████████▋ | 33974/45917 [12:43:53<3:41:13, 1.11s/it][2025-04-24 06:06:04] (step=0033975) Train Loss: 6.3548, Train Steps/Sec: 0.90 + 74%|█████████████████████████████████████████████████████████████████████████████▋ | 33999/45917 [12:44:20<3:37:37, 1.10s/it][2025-04-24 06:06:32] (step=0034000) Train Loss: 6.3510, Train Steps/Sec: 0.90 +[2025-04-24 06:06:32] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 06:06:32] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 06:06:32] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.84s/it] +[2025-04-24 06:11:17] Finish Eval in 34000 steps...█████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.49s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 06:11:40] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0034000.pt +[2025-04-24 06:11:42] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0032000.pt + 74%|█████████████████████████████████████████████████████████████████████████████▊ | 34024/45917 [12:49:59<3:48:06, 1.15s/it][2025-04-24 06:12:10] (step=0034025) Train Loss: 6.3601, Train Steps/Sec: 0.07 + 74%|█████████████████████████████████████████████████████████████████████████████▊ | 34049/45917 [12:50:26<3:40:29, 1.11s/it][2025-04-24 06:12:38] (step=0034050) Train Loss: 6.3639, Train Steps/Sec: 0.90 + 74%|█████████████████████████████████████████████████████████████████████████████▉ | 34074/45917 [12:50:54<3:36:19, 1.10s/it][2025-04-24 06:13:05] (step=0034075) Train Loss: 6.3907, Train Steps/Sec: 0.90 + 74%|█████████████████████████████████████████████████████████████████████████████▉ | 34099/45917 [12:51:22<3:37:58, 1.11s/it][2025-04-24 06:13:33] (step=0034100) Train Loss: 6.3558, Train Steps/Sec: 0.90 + 74%|██████████████████████████████████████████████████████████████████████████████ | 34124/45917 [12:51:50<3:41:51, 1.13s/it][2025-04-24 06:14:01] (step=0034125) Train Loss: 6.3946, Train Steps/Sec: 0.90 + 74%|██████████████████████████████████████████████████████████████████████████████ | 34149/45917 [12:52:17<3:37:23, 1.11s/it][2025-04-24 06:14:29] (step=0034150) Train Loss: 6.3517, Train Steps/Sec: 0.90 + 74%|██████████████████████████████████████████████████████████████████████████████▏ | 34174/45917 [12:52:45<3:36:46, 1.11s/it][2025-04-24 06:14:56] (step=0034175) Train Loss: 6.3789, Train Steps/Sec: 0.90 + 74%|██████████████████████████████████████████████████████████████████████████████▏ | 34199/45917 [12:53:13<3:33:23, 1.09s/it][2025-04-24 06:15:24] (step=0034200) Train Loss: 6.3319, Train Steps/Sec: 0.90 + 75%|██████████████████████████████████████████████████████████████████████████████▎ | 34224/45917 [12:53:41<3:39:04, 1.12s/it][2025-04-24 06:15:52] (step=0034225) Train Loss: 6.3673, Train Steps/Sec: 0.91 + 75%|██████████████████████████████████████████████████████████████████████████████▎ | 34249/45917 [12:54:08<3:37:31, 1.12s/it][2025-04-24 06:16:19] (step=0034250) Train Loss: 6.3341, Train Steps/Sec: 0.90 + 75%|██████████████████████████████████████████████████████████████████████████████▍ | 34274/45917 [12:54:36<3:35:42, 1.11s/it][2025-04-24 06:16:48] (step=0034275) Train Loss: 6.3798, Train Steps/Sec: 0.90 + 75%|██████████████████████████████████████████████████████████████████████████████▍ | 34299/45917 [12:55:04<3:31:53, 1.09s/it][2025-04-24 06:17:15] (step=0034300) Train Loss: 6.3365, Train Steps/Sec: 0.90 + 75%|██████████████████████████████████████████████████████████████████████████████▍ | 34324/45917 [12:55:32<3:36:59, 1.12s/it][2025-04-24 06:17:43] (step=0034325) Train Loss: 6.3631, Train Steps/Sec: 0.91 + 75%|██████████████████████████████████████████████████████████████████████████████▌ | 34349/45917 [12:56:00<3:36:15, 1.12s/it][2025-04-24 06:18:11] (step=0034350) Train Loss: 6.3254, Train Steps/Sec: 0.90 + 75%|██████████████████████████████████████████████████████████████████████████████▌ | 34374/45917 [12:56:27<3:31:30, 1.10s/it][2025-04-24 06:18:38] (step=0034375) Train Loss: 6.3373, Train Steps/Sec: 0.90 + 75%|██████████████████████████████████████████████████████████████████████████████▋ | 34399/45917 [12:56:55<3:30:01, 1.09s/it][2025-04-24 06:19:06] (step=0034400) Train Loss: 6.3570, Train Steps/Sec: 0.90 + 75%|██████████████████████████████████████████████████████████████████████████████▋ | 34424/45917 [12:57:23<3:36:02, 1.13s/it][2025-04-24 06:19:34] (step=0034425) Train Loss: 6.3646, Train Steps/Sec: 0.90 + 75%|██████████████████████████████████████████████████████████████████████████████▊ | 34449/45917 [12:57:50<3:30:19, 1.10s/it][2025-04-24 06:20:02] (step=0034450) Train Loss: 6.3653, Train Steps/Sec: 0.91 + 75%|██████████████████████████████████████████████████████████████████████████████▊ | 34474/45917 [12:58:18<3:31:20, 1.11s/it][2025-04-24 06:20:29] (step=0034475) Train Loss: 6.3809, Train Steps/Sec: 0.90 + 75%|██████████████████████████████████████████████████████████████████████████████▉ | 34499/45917 [12:58:46<3:27:59, 1.09s/it][2025-04-24 06:20:57] (step=0034500) Train Loss: 6.3478, Train Steps/Sec: 0.90 + 75%|██████████████████████████████████████████████████████████████████████████████▉ | 34524/45917 [12:59:14<3:34:16, 1.13s/it][2025-04-24 06:21:25] (step=0034525) Train Loss: 6.3252, Train Steps/Sec: 0.90 + 75%|███████████████████████████████████████████████████████████████████████████████ | 34549/45917 [12:59:41<3:29:14, 1.10s/it][2025-04-24 06:21:53] (step=0034550) Train Loss: 6.3385, Train Steps/Sec: 0.90 + 75%|███████████████████████████████████████████████████████████████████████████████ | 34574/45917 [13:00:09<3:29:09, 1.11s/it][2025-04-24 06:22:20] (step=0034575) Train Loss: 6.3639, Train Steps/Sec: 0.90 + 75%|███████████████████████████████████████████████████████████████████████████████ | 34599/45917 [13:00:37<3:26:49, 1.10s/it][2025-04-24 06:22:48] (step=0034600) Train Loss: 6.3730, Train Steps/Sec: 0.90 + 75%|███████████████████████████████████████████████████████████████████████████████▏ | 34624/45917 [13:01:04<3:32:09, 1.13s/it][2025-04-24 06:23:16] (step=0034625) Train Loss: 6.2814, Train Steps/Sec: 0.91 + 75%|███████████████████████████████████████████████████████████████████████████████▏ | 34649/45917 [13:01:32<3:28:19, 1.11s/it][2025-04-24 06:23:43] (step=0034650) Train Loss: 6.3509, Train Steps/Sec: 0.91 + 76%|███████████████████████████████████████████████████████████████████████████████▎ | 34674/45917 [13:02:00<3:26:47, 1.10s/it][2025-04-24 06:24:11] (step=0034675) Train Loss: 6.3844, Train Steps/Sec: 0.90 + 76%|███████████████████████████████████████████████████████████████████████████████▎ | 34699/45917 [13:02:27<3:23:49, 1.09s/it][2025-04-24 06:24:39] (step=0034700) Train Loss: 6.3504, Train Steps/Sec: 0.90 + 76%|███████████████████████████████████████████████████████████████████████████████▍ | 34724/45917 [13:02:55<3:29:27, 1.12s/it][2025-04-24 06:25:06] (step=0034725) Train Loss: 6.3604, Train Steps/Sec: 0.91 + 76%|███████████████████████████████████████████████████████████████████████████████▍ | 34749/45917 [13:03:23<3:26:09, 1.11s/it][2025-04-24 06:25:34] (step=0034750) Train Loss: 6.3778, Train Steps/Sec: 0.90 + 76%|███████████████████████████████████████████████████████████████████████████████▌ | 34774/45917 [13:03:50<3:23:41, 1.10s/it][2025-04-24 06:26:02] (step=0034775) Train Loss: 6.3983, Train Steps/Sec: 0.91 + 76%|███████████████████████████████████████████████████████████████████████████████▌ | 34799/45917 [13:04:18<3:22:03, 1.09s/it][2025-04-24 06:26:29] (step=0034800) Train Loss: 6.3744, Train Steps/Sec: 0.90 + 76%|███████████████████████████████████████████████████████████████████████████████▋ | 34824/45917 [13:04:46<3:29:02, 1.13s/it][2025-04-24 06:26:57] (step=0034825) Train Loss: 6.3429, Train Steps/Sec: 0.90 + 76%|███████████████████████████████████████████████████████████████████████████████▋ | 34849/45917 [13:05:13<3:24:23, 1.11s/it][2025-04-24 06:27:25] (step=0034850) Train Loss: 6.3241, Train Steps/Sec: 0.90 + 76%|███████████████████████████████████████████████████████████████████████████████▋ | 34874/45917 [13:05:41<3:22:16, 1.10s/it][2025-04-24 06:27:53] (step=0034875) Train Loss: 6.3741, Train Steps/Sec: 0.90 + 76%|███████████████████████████████████████████████████████████████████████████████▊ | 34899/45917 [13:06:09<3:21:57, 1.10s/it][2025-04-24 06:28:20] (step=0034900) Train Loss: 6.3977, Train Steps/Sec: 0.90 + 76%|███████████████████████████████████████████████████████████████████████████████▊ | 34924/45917 [13:06:37<3:26:28, 1.13s/it][2025-04-24 06:28:48] (step=0034925) Train Loss: 6.3384, Train Steps/Sec: 0.90 + 76%|███████████████████████████████████████████████████████████████████████████████▉ | 34949/45917 [13:07:05<3:22:58, 1.11s/it][2025-04-24 06:29:16] (step=0034950) Train Loss: 6.3727, Train Steps/Sec: 0.91 + 76%|███████████████████████████████████████████████████████████████████████████████▉ | 34974/45917 [13:07:32<3:20:02, 1.10s/it][2025-04-24 06:29:43] (step=0034975) Train Loss: 6.3870, Train Steps/Sec: 0.91 + 76%|████████████████████████████████████████████████████████████████████████████████ | 34999/45917 [13:08:00<3:19:05, 1.09s/it][2025-04-24 06:30:11] (step=0035000) Train Loss: 6.3812, Train Steps/Sec: 0.90 + 76%|████████████████████████████████████████████████████████████████████████████████ | 35024/45917 [13:08:27<3:23:46, 1.12s/it][2025-04-24 06:30:39] (step=0035025) Train Loss: 6.3870, Train Steps/Sec: 0.91 + 76%|████████████████████████████████████████████████████████████████████████████████▏ | 35049/45917 [13:08:55<3:20:23, 1.11s/it][2025-04-24 06:31:06] (step=0035050) Train Loss: 6.4004, Train Steps/Sec: 0.90 + 76%|████████████████████████████████████████████████████████████████████████████████▏ | 35074/45917 [13:09:23<3:19:44, 1.11s/it][2025-04-24 06:31:34] (step=0035075) Train Loss: 6.3782, Train Steps/Sec: 0.90 + 76%|████████████████████████████████████████████████████████████████████████████████▎ | 35099/45917 [13:09:50<3:16:04, 1.09s/it][2025-04-24 06:32:02] (step=0035100) Train Loss: 6.3614, Train Steps/Sec: 0.90 + 76%|████████████████████████████████████████████████████████████████████████████████▎ | 35124/45917 [13:10:18<3:21:27, 1.12s/it][2025-04-24 06:32:29] (step=0035125) Train Loss: 6.3384, Train Steps/Sec: 0.91 + 77%|████████████████████████████████████████████████████████████████████████████████▍ | 35149/45917 [13:10:46<3:20:04, 1.11s/it][2025-04-24 06:32:57] (step=0035150) Train Loss: 6.3534, Train Steps/Sec: 0.90 + 77%|████████████████████████████████████████████████████████████████████████████████▍ | 35174/45917 [13:11:14<3:17:08, 1.10s/it][2025-04-24 06:33:25] (step=0035175) Train Loss: 6.3649, Train Steps/Sec: 0.90 + 77%|████████████████████████████████████████████████████████████████████████████████▍ | 35199/45917 [13:11:41<3:15:07, 1.09s/it][2025-04-24 06:33:53] (step=0035200) Train Loss: 6.3312, Train Steps/Sec: 0.90 + 77%|████████████████████████████████████████████████████████████████████████████████▌ | 35224/45917 [13:12:09<3:23:39, 1.14s/it][2025-04-24 06:34:20] (step=0035225) Train Loss: 6.3479, Train Steps/Sec: 0.90 + 77%|████████████████████████████████████████████████████████████████████████████████▌ | 35249/45917 [13:12:37<3:16:39, 1.11s/it][2025-04-24 06:34:48] (step=0035250) Train Loss: 6.3960, Train Steps/Sec: 0.90 + 77%|████████████████████████████████████████████████████████████████████████████████▋ | 35274/45917 [13:13:05<3:14:33, 1.10s/it][2025-04-24 06:35:16] (step=0035275) Train Loss: 6.3586, Train Steps/Sec: 0.90 + 77%|████████████████████████████████████████████████████████████████████████████████▋ | 35299/45917 [13:13:32<3:13:24, 1.09s/it][2025-04-24 06:35:44] (step=0035300) Train Loss: 6.3367, Train Steps/Sec: 0.90 + 77%|████████████████████████████████████████████████████████████████████████████████▊ | 35324/45917 [13:14:00<3:18:28, 1.12s/it][2025-04-24 06:36:11] (step=0035325) Train Loss: 6.3790, Train Steps/Sec: 0.91 + 77%|████████████████████████████████████████████████████████████████████████████████▊ | 35349/45917 [13:14:28<3:16:05, 1.11s/it][2025-04-24 06:36:39] (step=0035350) Train Loss: 6.3144, Train Steps/Sec: 0.90 + 77%|████████████████████████████████████████████████████████████████████████████████▉ | 35374/45917 [13:14:55<3:13:45, 1.10s/it][2025-04-24 06:37:07] (step=0035375) Train Loss: 6.3516, Train Steps/Sec: 0.90 + 77%|████████████████████████████████████████████████████████████████████████████████▉ | 35399/45917 [13:15:23<3:11:06, 1.09s/it][2025-04-24 06:37:34] (step=0035400) Train Loss: 6.3047, Train Steps/Sec: 0.90 + 77%|█████████████████████████████████████████████████████████████████████████████████ | 35424/45917 [13:15:51<3:16:02, 1.12s/it][2025-04-24 06:38:02] (step=0035425) Train Loss: 6.3175, Train Steps/Sec: 0.90 + 77%|█████████████████████████████████████████████████████████████████████████████████ | 35449/45917 [13:16:19<3:13:27, 1.11s/it][2025-04-24 06:38:30] (step=0035450) Train Loss: 6.3355, Train Steps/Sec: 0.90 + 77%|█████████████████████████████████████████████████████████████████████████████████ | 35474/45917 [13:16:46<3:11:46, 1.10s/it][2025-04-24 06:38:58] (step=0035475) Train Loss: 6.3582, Train Steps/Sec: 0.90 + 77%|█████████████████████████████████████████████████████████████████████████████████▏ | 35499/45917 [13:17:14<3:10:06, 1.09s/it][2025-04-24 06:39:25] (step=0035500) Train Loss: 6.3598, Train Steps/Sec: 0.90 + 77%|█████████████████████████████████████████████████████████████████████████████████▏ | 35524/45917 [13:17:42<3:14:28, 1.12s/it][2025-04-24 06:39:53] (step=0035525) Train Loss: 6.3610, Train Steps/Sec: 0.91 + 77%|█████████████████████████████████████████████████████████████████████████████████▎ | 35549/45917 [13:18:09<3:10:01, 1.10s/it][2025-04-24 06:40:21] (step=0035550) Train Loss: 6.3532, Train Steps/Sec: 0.91 + 77%|█████████████████████████████████████████████████████████████████████████████████▎ | 35574/45917 [13:18:37<3:11:14, 1.11s/it][2025-04-24 06:40:48] (step=0035575) Train Loss: 6.4181, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████▍ | 35599/45917 [13:19:05<3:07:33, 1.09s/it][2025-04-24 06:41:16] (step=0035600) Train Loss: 6.3913, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████▍ | 35624/45917 [13:19:33<3:12:51, 1.12s/it][2025-04-24 06:41:44] (step=0035625) Train Loss: 6.3549, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████▌ | 35649/45917 [13:20:00<3:11:35, 1.12s/it][2025-04-24 06:42:12] (step=0035650) Train Loss: 6.3783, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████▌ | 35674/45917 [13:20:28<3:08:27, 1.10s/it][2025-04-24 06:42:39] (step=0035675) Train Loss: 6.3865, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████▋ | 35699/45917 [13:20:56<3:08:48, 1.11s/it][2025-04-24 06:43:07] (step=0035700) Train Loss: 6.3234, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████▋ | 35724/45917 [13:21:23<3:12:18, 1.13s/it][2025-04-24 06:43:35] (step=0035725) Train Loss: 6.3815, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████▋ | 35749/45917 [13:21:51<3:06:45, 1.10s/it][2025-04-24 06:44:02] (step=0035750) Train Loss: 6.3758, Train Steps/Sec: 0.91 + 78%|█████████████████████████████████████████████████████████████████████████████████▊ | 35774/45917 [13:22:19<3:05:34, 1.10s/it][2025-04-24 06:44:30] (step=0035775) Train Loss: 6.3480, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████▊ | 35799/45917 [13:22:46<3:04:55, 1.10s/it][2025-04-24 06:44:58] (step=0035800) Train Loss: 6.3293, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████▉ | 35824/45917 [13:23:14<3:10:03, 1.13s/it][2025-04-24 06:45:25] (step=0035825) Train Loss: 6.3520, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████▉ | 35849/45917 [13:23:42<3:05:04, 1.10s/it][2025-04-24 06:45:53] (step=0035850) Train Loss: 6.2928, Train Steps/Sec: 0.90 + 78%|██████████████████████████████████████████████████████████████████████████████████ | 35874/45917 [13:24:09<3:03:07, 1.09s/it][2025-04-24 06:46:21] (step=0035875) Train Loss: 6.4019, Train Steps/Sec: 0.90 + 78%|██████████████████████████████████████████████████████████████████████████████████ | 35899/45917 [13:24:48<5:16:46, 1.90s/it][2025-04-24 06:46:59] (step=0035900) Train Loss: 6.3696, Train Steps/Sec: 0.65 + 78%|██████████████████████████████████████████████████████████████████████████████████▏ | 35924/45917 [13:25:16<3:07:02, 1.12s/it][2025-04-24 06:47:27] (step=0035925) Train Loss: 6.3343, Train Steps/Sec: 0.90 + 78%|██████████████████████████████████████████████████████████████████████████████████▏ | 35949/45917 [13:25:43<3:03:43, 1.11s/it][2025-04-24 06:47:55] (step=0035950) Train Loss: 6.3515, Train Steps/Sec: 0.90 + 78%|██████████████████████████████████████████████████████████████████████████████████▎ | 35974/45917 [13:26:11<3:03:07, 1.11s/it][2025-04-24 06:48:22] (step=0035975) Train Loss: 6.3758, Train Steps/Sec: 0.90 + 78%|██████████████████████████████████████████████████████████████████████████████████▎ | 35999/45917 [13:26:39<2:59:50, 1.09s/it][2025-04-24 06:48:50] (step=0036000) Train Loss: 6.3575, Train Steps/Sec: 0.90 +[2025-04-24 06:48:50] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 06:48:50] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 06:48:50] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.68s/it] +[2025-04-24 06:53:35] Finish Eval in 36000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.22s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 06:53:58] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0036000.pt +[2025-04-24 06:54:00] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0034000.pt + 78%|██████████████████████████████████████████████████████████████████████████████████▍ | 36024/45917 [13:32:17<3:09:26, 1.15s/it][2025-04-24 06:54:28] (step=0036025) Train Loss: 6.3842, Train Steps/Sec: 0.07 + 79%|██████████████████████████████████████████████████████████████████████████████████▍ | 36049/45917 [13:32:44<3:00:52, 1.10s/it][2025-04-24 06:54:56] (step=0036050) Train Loss: 6.3396, Train Steps/Sec: 0.90 + 79%|██████████████████████████████████████████████████████████████████████████████████▍ | 36074/45917 [13:33:12<3:00:29, 1.10s/it][2025-04-24 06:55:23] (step=0036075) Train Loss: 6.4003, Train Steps/Sec: 0.91 + 79%|██████████████████████████████████████████████████████████████████████████████████▌ | 36099/45917 [13:33:46<3:00:11, 1.10s/it][2025-04-24 06:55:57] (step=0036100) Train Loss: 6.3789, Train Steps/Sec: 0.74 + 79%|██████████████████████████████████████████████████████████████████████████████████▌ | 36124/45917 [13:34:14<3:02:48, 1.12s/it][2025-04-24 06:56:31] (step=0036125) Train Loss: 6.3445, Train Steps/Sec: 0.73 + 79%|██████████████████████████████████████████████████████████████████████████████████▋ | 36149/45917 [13:34:48<3:00:37, 1.11s/it][2025-04-24 06:57:05] (step=0036150) Train Loss: 6.3572, Train Steps/Sec: 0.74 + 79%|██████████████████████████████████████████████████████████████████████████████████▋ | 36174/45917 [13:35:22<3:00:33, 1.11s/it][2025-04-24 06:57:33] (step=0036175) Train Loss: 6.3272, Train Steps/Sec: 0.90 + 79%|██████████████████████████████████████████████████████████████████████████████████▊ | 36199/45917 [13:35:49<2:55:24, 1.08s/it][2025-04-24 06:58:01] (step=0036200) Train Loss: 6.3711, Train Steps/Sec: 0.90 + 79%|██████████████████████████████████████████████████████████████████████████████████▊ | 36224/45917 [13:36:17<3:01:59, 1.13s/it][2025-04-24 06:58:28] (step=0036225) Train Loss: 6.3806, Train Steps/Sec: 0.90 + 79%|██████████████████████████████████████████████████████████████████████████████████▉ | 36249/45917 [13:36:45<2:58:40, 1.11s/it][2025-04-24 06:58:56] (step=0036250) Train Loss: 6.3551, Train Steps/Sec: 0.90 + 79%|██████████████████████████████████████████████████████████████████████████████████▉ | 36274/45917 [13:37:13<2:57:15, 1.10s/it][2025-04-24 06:59:24] (step=0036275) Train Loss: 6.4006, Train Steps/Sec: 0.90 + 79%|███████████████████████████████████████████████████████████████████████████████████ | 36299/45917 [13:37:40<2:55:40, 1.10s/it][2025-04-24 06:59:52] (step=0036300) Train Loss: 6.3853, Train Steps/Sec: 0.90 + 79%|███████████████████████████████████████████████████████████████████████████████████ | 36324/45917 [13:38:08<3:00:00, 1.13s/it][2025-04-24 07:00:20] (step=0036325) Train Loss: 6.3159, Train Steps/Sec: 0.90 + 79%|███████████████████████████████████████████████████████████████████████████████████ | 36349/45917 [13:38:36<2:56:54, 1.11s/it][2025-04-24 07:00:47] (step=0036350) Train Loss: 6.3422, Train Steps/Sec: 0.90 + 79%|███████████████████████████████████████████████████████████████████████████████████▏ | 36374/45917 [13:39:04<2:54:49, 1.10s/it][2025-04-24 07:01:15] (step=0036375) Train Loss: 6.3540, Train Steps/Sec: 0.91 + 79%|███████████████████████████████████████████████████████████████████████████████████▏ | 36399/45917 [13:39:31<2:53:54, 1.10s/it][2025-04-24 07:01:43] (step=0036400) Train Loss: 6.3816, Train Steps/Sec: 0.90 + 79%|███████████████████████████████████████████████████████████████████████████████████▎ | 36424/45917 [13:39:59<2:57:41, 1.12s/it][2025-04-24 07:02:10] (step=0036425) Train Loss: 6.3549, Train Steps/Sec: 0.90 + 79%|███████████████████████████████████████████████████████████████████████████████████▎ | 36449/45917 [13:40:27<2:54:05, 1.10s/it][2025-04-24 07:02:38] (step=0036450) Train Loss: 6.3192, Train Steps/Sec: 0.90 + 79%|███████████████████████████████████████████████████████████████████████████████████▍ | 36474/45917 [13:40:55<2:52:58, 1.10s/it][2025-04-24 07:03:06] (step=0036475) Train Loss: 6.3444, Train Steps/Sec: 0.90 + 79%|███████████████████████████████████████████████████████████████████████████████████▍ | 36499/45917 [13:41:22<2:51:16, 1.09s/it][2025-04-24 07:03:34] (step=0036500) Train Loss: 6.3782, Train Steps/Sec: 0.90 + 80%|███████████████████████████████████████████████████████████████████████████████████▌ | 36524/45917 [13:41:50<2:55:11, 1.12s/it][2025-04-24 07:04:01] (step=0036525) Train Loss: 6.3386, Train Steps/Sec: 0.91 + 80%|███████████████████████████████████████████████████████████████████████████████████▌ | 36549/45917 [13:42:18<2:52:38, 1.11s/it][2025-04-24 07:04:29] (step=0036550) Train Loss: 6.3825, Train Steps/Sec: 0.90 + 80%|███████████████████████████████████████████████████████████████████████████████████▋ | 36574/45917 [13:42:46<2:51:56, 1.10s/it][2025-04-24 07:04:57] (step=0036575) Train Loss: 6.3752, Train Steps/Sec: 0.90 + 80%|███████████████████████████████████████████████████████████████████████████████████▋ | 36599/45917 [13:43:13<2:50:27, 1.10s/it][2025-04-24 07:05:25] (step=0036600) Train Loss: 6.3517, Train Steps/Sec: 0.90 + 80%|███████████████████████████████████████████████████████████████████████████████████▋ | 36624/45917 [13:43:41<2:54:48, 1.13s/it][2025-04-24 07:05:52] (step=0036625) Train Loss: 6.3638, Train Steps/Sec: 0.90 + 80%|███████████████████████████████████████████████████████████████████████████████████▊ | 36649/45917 [13:44:09<2:51:42, 1.11s/it][2025-04-24 07:06:20] (step=0036650) Train Loss: 6.3563, Train Steps/Sec: 0.91 + 80%|███████████████████████████████████████████████████████████████████████████████████▊ | 36674/45917 [13:44:36<2:49:56, 1.10s/it][2025-04-24 07:06:48] (step=0036675) Train Loss: 6.3934, Train Steps/Sec: 0.90 + 80%|███████████████████████████████████████████████████████████████████████████████████▉ | 36699/45917 [13:45:04<2:48:02, 1.09s/it][2025-04-24 07:07:15] (step=0036700) Train Loss: 6.3592, Train Steps/Sec: 0.90 + 80%|███████████████████████████████████████████████████████████████████████████████████▉ | 36724/45917 [13:45:32<2:52:18, 1.12s/it][2025-04-24 07:07:43] (step=0036725) Train Loss: 6.3628, Train Steps/Sec: 0.90 + 80%|████████████████████████████████████████████████████████████████████████████████████ | 36749/45917 [13:46:00<2:50:52, 1.12s/it][2025-04-24 07:08:11] (step=0036750) Train Loss: 6.3722, Train Steps/Sec: 0.90 + 80%|████████████████████████████████████████████████████████████████████████████████████ | 36774/45917 [13:46:28<2:48:48, 1.11s/it][2025-04-24 07:08:39] (step=0036775) Train Loss: 6.3296, Train Steps/Sec: 0.90 + 80%|████████████████████████████████████████████████████████████████████████████████████▏ | 36799/45917 [13:46:55<2:47:29, 1.10s/it][2025-04-24 07:09:07] (step=0036800) Train Loss: 6.3367, Train Steps/Sec: 0.90 + 80%|████████████████████████████████████████████████████████████████████████████████████▏ | 36824/45917 [13:47:23<2:50:35, 1.13s/it][2025-04-24 07:09:34] (step=0036825) Train Loss: 6.3243, Train Steps/Sec: 0.90 + 80%|████████████████████████████████████████████████████████████████████████████████████▎ | 36849/45917 [13:47:51<2:48:17, 1.11s/it][2025-04-24 07:10:02] (step=0036850) Train Loss: 6.3485, Train Steps/Sec: 0.90 + 80%|████████████████████████████████████████████████████████████████████████████████████▎ | 36874/45917 [13:48:19<2:44:33, 1.09s/it][2025-04-24 07:10:30] (step=0036875) Train Loss: 6.3410, Train Steps/Sec: 0.91 + 80%|████████████████████████████████████████████████████████████████████████████████████▍ | 36899/45917 [13:48:46<2:45:13, 1.10s/it][2025-04-24 07:10:58] (step=0036900) Train Loss: 6.3742, Train Steps/Sec: 0.90 + 80%|████████████████████████████████████████████████████████████████████████████████████▍ | 36924/45917 [13:49:14<2:48:07, 1.12s/it][2025-04-24 07:11:25] (step=0036925) Train Loss: 6.3902, Train Steps/Sec: 0.91 + 80%|████████████████████████████████████████████████████████████████████████████████████▍ | 36949/45917 [13:49:42<2:46:45, 1.12s/it][2025-04-24 07:11:53] (step=0036950) Train Loss: 6.3619, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████▌ | 36974/45917 [13:50:09<2:43:34, 1.10s/it][2025-04-24 07:12:21] (step=0036975) Train Loss: 6.3992, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████▌ | 36999/45917 [13:50:37<2:43:36, 1.10s/it][2025-04-24 07:12:49] (step=0037000) Train Loss: 6.4104, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████▋ | 37024/45917 [13:51:05<2:46:59, 1.13s/it][2025-04-24 07:13:16] (step=0037025) Train Loss: 6.3330, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████▋ | 37049/45917 [13:51:33<2:44:18, 1.11s/it][2025-04-24 07:13:44] (step=0037050) Train Loss: 6.3652, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████▊ | 37074/45917 [13:52:00<2:42:14, 1.10s/it][2025-04-24 07:14:12] (step=0037075) Train Loss: 6.3519, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████▊ | 37099/45917 [13:52:28<2:40:09, 1.09s/it][2025-04-24 07:14:39] (step=0037100) Train Loss: 6.3874, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████▉ | 37124/45917 [13:52:56<2:44:42, 1.12s/it][2025-04-24 07:15:07] (step=0037125) Train Loss: 6.3453, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████▉ | 37149/45917 [13:53:24<2:43:22, 1.12s/it][2025-04-24 07:15:35] (step=0037150) Train Loss: 6.3671, Train Steps/Sec: 0.90 + 81%|█████████████████████████████████████████████████████████████████████████████████████ | 37174/45917 [13:53:51<2:40:09, 1.10s/it][2025-04-24 07:16:03] (step=0037175) Train Loss: 6.3468, Train Steps/Sec: 0.90 + 81%|█████████████████████████████████████████████████████████████████████████████████████ | 37199/45917 [13:54:19<2:38:15, 1.09s/it][2025-04-24 07:16:30] (step=0037200) Train Loss: 6.3234, Train Steps/Sec: 0.90 + 81%|█████████████████████████████████████████████████████████████████████████████████████ | 37224/45917 [13:54:47<2:43:24, 1.13s/it][2025-04-24 07:16:58] (step=0037225) Train Loss: 6.3404, Train Steps/Sec: 0.90 + 81%|█████████████████████████████████████████████████████████████████████████████████████▏ | 37249/45917 [13:55:14<2:39:26, 1.10s/it][2025-04-24 07:17:26] (step=0037250) Train Loss: 6.3453, Train Steps/Sec: 0.90 + 81%|█████████████████████████████████████████████████████████████████████████████████████▏ | 37274/45917 [13:55:42<2:39:58, 1.11s/it][2025-04-24 07:17:53] (step=0037275) Train Loss: 6.3386, Train Steps/Sec: 0.90 + 81%|█████████████████████████████████████████████████████████████████████████████████████▎ | 37299/45917 [13:56:10<2:37:10, 1.09s/it][2025-04-24 07:18:21] (step=0037300) Train Loss: 6.3995, Train Steps/Sec: 0.90 + 81%|█████████████████████████████████████████████████████████████████████████████████████▎ | 37324/45917 [13:56:38<2:41:35, 1.13s/it][2025-04-24 07:18:49] (step=0037325) Train Loss: 6.3203, Train Steps/Sec: 0.90 + 81%|█████████████████████████████████████████████████████████████████████████████████████▍ | 37349/45917 [13:57:06<2:38:35, 1.11s/it][2025-04-24 07:19:17] (step=0037350) Train Loss: 6.3754, Train Steps/Sec: 0.90 + 81%|█████████████████████████████████████████████████████████████████████████████████████▍ | 37374/45917 [13:57:33<2:37:40, 1.11s/it][2025-04-24 07:19:44] (step=0037375) Train Loss: 6.3776, Train Steps/Sec: 0.90 + 81%|█████████████████████████████████████████████████████████████████████████████████████▌ | 37399/45917 [13:58:01<2:36:00, 1.10s/it][2025-04-24 07:20:12] (step=0037400) Train Loss: 6.3238, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████▌ | 37424/45917 [13:58:29<2:39:47, 1.13s/it][2025-04-24 07:20:40] (step=0037425) Train Loss: 6.3512, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████▋ | 37449/45917 [13:58:57<2:36:10, 1.11s/it][2025-04-24 07:21:08] (step=0037450) Train Loss: 6.3854, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████▋ | 37474/45917 [13:59:24<2:34:45, 1.10s/it][2025-04-24 07:21:35] (step=0037475) Train Loss: 6.3585, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████▊ | 37499/45917 [13:59:52<2:33:45, 1.10s/it][2025-04-24 07:22:03] (step=0037500) Train Loss: 6.3745, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████▊ | 37524/45917 [14:00:20<2:36:34, 1.12s/it][2025-04-24 07:22:31] (step=0037525) Train Loss: 6.3419, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████▊ | 37549/45917 [14:00:47<2:33:39, 1.10s/it][2025-04-24 07:22:59] (step=0037550) Train Loss: 6.3446, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████▉ | 37574/45917 [14:01:15<2:33:52, 1.11s/it][2025-04-24 07:23:26] (step=0037575) Train Loss: 6.3548, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████▉ | 37599/45917 [14:01:55<8:10:57, 3.54s/it][2025-04-24 07:24:06] (step=0037600) Train Loss: 6.3683, Train Steps/Sec: 0.63 + 82%|██████████████████████████████████████████████████████████████████████████████████████ | 37624/45917 [14:02:22<2:36:07, 1.13s/it][2025-04-24 07:24:33] (step=0037625) Train Loss: 6.3506, Train Steps/Sec: 0.91 + 82%|██████████████████████████████████████████████████████████████████████████████████████ | 37649/45917 [14:02:58<2:32:13, 1.10s/it][2025-04-24 07:25:09] (step=0037650) Train Loss: 6.3729, Train Steps/Sec: 0.70 + 82%|██████████████████████████████████████████████████████████████████████████████████████▏ | 37674/45917 [14:03:26<2:32:50, 1.11s/it][2025-04-24 07:25:37] (step=0037675) Train Loss: 6.3740, Train Steps/Sec: 0.90 + 82%|██████████████████████████████████████████████████████████████████████████████████████▏ | 37699/45917 [14:03:54<2:29:28, 1.09s/it][2025-04-24 07:26:05] (step=0037700) Train Loss: 6.3538, Train Steps/Sec: 0.90 + 82%|██████████████████████████████████████████████████████████████████████████████████████▎ | 37724/45917 [14:04:22<2:33:46, 1.13s/it][2025-04-24 07:26:33] (step=0037725) Train Loss: 6.3383, Train Steps/Sec: 0.90 + 82%|██████████████████████████████████████████████████████████████████████████████████████▎ | 37749/45917 [14:04:49<2:30:19, 1.10s/it][2025-04-24 07:27:00] (step=0037750) Train Loss: 6.3446, Train Steps/Sec: 0.91 + 82%|██████████████████████████████████████████████████████████████████████████████████████▍ | 37774/45917 [14:05:17<2:30:14, 1.11s/it][2025-04-24 07:27:28] (step=0037775) Train Loss: 6.3881, Train Steps/Sec: 0.90 + 82%|██████████████████████████████████████████████████████████████████████████████████████▍ | 37799/45917 [14:05:45<2:27:26, 1.09s/it][2025-04-24 07:27:56] (step=0037800) Train Loss: 6.3573, Train Steps/Sec: 0.90 + 82%|██████████████████████████████████████████████████████████████████████████████████████▍ | 37824/45917 [14:06:12<2:31:36, 1.12s/it][2025-04-24 07:28:23] (step=0037825) Train Loss: 6.3412, Train Steps/Sec: 0.90 + 82%|██████████████████████████████████████████████████████████████████████████████████████▌ | 37849/45917 [14:06:40<2:29:47, 1.11s/it][2025-04-24 07:28:51] (step=0037850) Train Loss: 6.3750, Train Steps/Sec: 0.90 + 82%|██████████████████████████████████████████████████████████████████████████████████████▌ | 37874/45917 [14:07:08<2:27:51, 1.10s/it][2025-04-24 07:29:19] (step=0037875) Train Loss: 6.4014, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████▋ | 37899/45917 [14:07:36<2:27:09, 1.10s/it][2025-04-24 07:29:47] (step=0037900) Train Loss: 6.3683, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████▋ | 37924/45917 [14:08:03<2:30:06, 1.13s/it][2025-04-24 07:30:15] (step=0037925) Train Loss: 6.3348, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████▊ | 37949/45917 [14:08:31<2:27:46, 1.11s/it][2025-04-24 07:30:42] (step=0037950) Train Loss: 6.3462, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████▊ | 37974/45917 [14:08:59<2:26:11, 1.10s/it][2025-04-24 07:31:10] (step=0037975) Train Loss: 6.3380, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████▉ | 37999/45917 [14:09:26<2:24:07, 1.09s/it][2025-04-24 07:31:38] (step=0038000) Train Loss: 6.3576, Train Steps/Sec: 0.90 +[2025-04-24 07:31:38] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 07:31:38] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 07:31:38] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.88s/it] +[2025-04-24 07:36:30] Finish Eval in 38000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.21s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 07:36:52] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0038000.pt +[2025-04-24 07:36:55] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0036000.pt + 83%|██████████████████████████████████████████████████████████████████████████████████████▉ | 38024/45917 [14:15:11<2:32:17, 1.16s/it][2025-04-24 07:37:23] (step=0038025) Train Loss: 6.3397, Train Steps/Sec: 0.07 + 83%|███████████████████████████████████████████████████████████████████████████████████████ | 38049/45917 [14:15:39<2:25:43, 1.11s/it][2025-04-24 07:37:50] (step=0038050) Train Loss: 6.3521, Train Steps/Sec: 0.91 + 83%|███████████████████████████████████████████████████████████████████████████████████████ | 38074/45917 [14:16:07<2:23:51, 1.10s/it][2025-04-24 07:38:18] (step=0038075) Train Loss: 6.3947, Train Steps/Sec: 0.90 + 83%|███████████████████████████████████████████████████████████████████████████████████████ | 38099/45917 [14:16:34<2:22:06, 1.09s/it][2025-04-24 07:38:46] (step=0038100) Train Loss: 6.3721, Train Steps/Sec: 0.90 + 83%|███████████████████████████████████████████████████████████████████████████████████████▏ | 38124/45917 [14:17:02<2:25:34, 1.12s/it][2025-04-24 07:39:13] (step=0038125) Train Loss: 6.3190, Train Steps/Sec: 0.90 + 83%|███████████████████████████████████████████████████████████████████████████████████████▏ | 38149/45917 [14:17:30<2:22:56, 1.10s/it][2025-04-24 07:39:41] (step=0038150) Train Loss: 6.3497, Train Steps/Sec: 0.91 + 83%|███████████████████████████████████████████████████████████████████████████████████████▎ | 38174/45917 [14:17:58<2:22:13, 1.10s/it][2025-04-24 07:40:09] (step=0038175) Train Loss: 6.3229, Train Steps/Sec: 0.90 + 83%|███████████████████████████████████████████████████████████████████████████████████████▎ | 38199/45917 [14:18:25<2:21:24, 1.10s/it][2025-04-24 07:40:37] (step=0038200) Train Loss: 6.3255, Train Steps/Sec: 0.90 + 83%|███████████████████████████████████████████████████████████████████████████████████████▍ | 38224/45917 [14:18:53<2:24:09, 1.12s/it][2025-04-24 07:41:04] (step=0038225) Train Loss: 6.3319, Train Steps/Sec: 0.90 + 83%|███████████████████████████████████████████████████████████████████████████████████████▍ | 38249/45917 [14:19:21<2:21:06, 1.10s/it][2025-04-24 07:41:32] (step=0038250) Train Loss: 6.3542, Train Steps/Sec: 0.91 + 83%|███████████████████████████████████████████████████████████████████████████████████████▌ | 38274/45917 [14:19:48<2:20:22, 1.10s/it][2025-04-24 07:42:00] (step=0038275) Train Loss: 6.4073, Train Steps/Sec: 0.90 + 83%|███████████████████████████████████████████████████████████████████████████████████████▌ | 38299/45917 [14:20:16<2:19:24, 1.10s/it][2025-04-24 07:42:27] (step=0038300) Train Loss: 6.3796, Train Steps/Sec: 0.90 + 83%|███████████████████████████████████████████████████████████████████████████████████████▋ | 38324/45917 [14:20:44<2:22:42, 1.13s/it][2025-04-24 07:42:55] (step=0038325) Train Loss: 6.3456, Train Steps/Sec: 0.90 + 84%|███████████████████████████████████████████████████████████████████████████████████████▋ | 38349/45917 [14:21:12<2:21:50, 1.12s/it][2025-04-24 07:43:23] (step=0038350) Train Loss: 6.3495, Train Steps/Sec: 0.90 + 84%|███████████████████████████████████████████████████████████████████████████████████████▊ | 38374/45917 [14:21:39<2:18:46, 1.10s/it][2025-04-24 07:43:51] (step=0038375) Train Loss: 6.3313, Train Steps/Sec: 0.90 + 84%|███████████████████████████████████████████████████████████████████████████████████████▊ | 38399/45917 [14:22:07<2:18:22, 1.10s/it][2025-04-24 07:44:18] (step=0038400) Train Loss: 6.3701, Train Steps/Sec: 0.90 + 84%|███████████████████████████████████████████████████████████████████████████████████████▊ | 38424/45917 [14:22:35<2:21:37, 1.13s/it][2025-04-24 07:44:46] (step=0038425) Train Loss: 6.3653, Train Steps/Sec: 0.90 + 84%|███████████████████████████████████████████████████████████████████████████████████████▉ | 38449/45917 [14:23:03<2:18:47, 1.12s/it][2025-04-24 07:45:14] (step=0038450) Train Loss: 6.3206, Train Steps/Sec: 0.90 + 84%|███████████████████████████████████████████████████████████████████████████████████████▉ | 38474/45917 [14:23:31<2:17:18, 1.11s/it][2025-04-24 07:45:42] (step=0038475) Train Loss: 6.3806, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████ | 38499/45917 [14:23:58<2:17:23, 1.11s/it][2025-04-24 07:46:10] (step=0038500) Train Loss: 6.3396, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████ | 38524/45917 [14:24:26<2:19:27, 1.13s/it][2025-04-24 07:46:37] (step=0038525) Train Loss: 6.3526, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████▏ | 38549/45917 [14:24:54<2:14:41, 1.10s/it][2025-04-24 07:47:05] (step=0038550) Train Loss: 6.3879, Train Steps/Sec: 0.91 + 84%|████████████████████████████████████████████████████████████████████████████████████████▏ | 38574/45917 [14:25:21<2:13:20, 1.09s/it][2025-04-24 07:47:32] (step=0038575) Train Loss: 6.3089, Train Steps/Sec: 0.91 + 84%|████████████████████████████████████████████████████████████████████████████████████████▎ | 38599/45917 [14:25:49<2:13:56, 1.10s/it][2025-04-24 07:48:00] (step=0038600) Train Loss: 6.3225, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████▎ | 38624/45917 [14:26:17<2:17:25, 1.13s/it][2025-04-24 07:48:28] (step=0038625) Train Loss: 6.3650, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████▍ | 38649/45917 [14:26:44<2:13:15, 1.10s/it][2025-04-24 07:48:55] (step=0038650) Train Loss: 6.3393, Train Steps/Sec: 0.91 + 84%|████████████████████████████████████████████████████████████████████████████████████████▍ | 38674/45917 [14:27:12<2:13:01, 1.10s/it][2025-04-24 07:49:23] (step=0038675) Train Loss: 6.3552, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████▍ | 38699/45917 [14:27:40<2:13:00, 1.11s/it][2025-04-24 07:49:51] (step=0038700) Train Loss: 6.3842, Train Steps/Sec: 0.89 + 84%|████████████████████████████████████████████████████████████████████████████████████████▌ | 38724/45917 [14:28:08<2:14:17, 1.12s/it][2025-04-24 07:50:19] (step=0038725) Train Loss: 6.3549, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████▌ | 38749/45917 [14:28:36<2:13:41, 1.12s/it][2025-04-24 07:50:47] (step=0038750) Train Loss: 6.3404, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████▋ | 38774/45917 [14:29:03<2:10:26, 1.10s/it][2025-04-24 07:51:15] (step=0038775) Train Loss: 6.3470, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████▋ | 38799/45917 [14:29:31<2:10:24, 1.10s/it][2025-04-24 07:51:42] (step=0038800) Train Loss: 6.3359, Train Steps/Sec: 0.90 + 85%|████████████████████████████████████████████████████████████████████████████████████████▊ | 38824/45917 [14:29:59<2:13:49, 1.13s/it][2025-04-24 07:52:10] (step=0038825) Train Loss: 6.3503, Train Steps/Sec: 0.90 + 85%|████████████████████████████████████████████████████████████████████████████████████████▊ | 38849/45917 [14:30:26<2:10:47, 1.11s/it][2025-04-24 07:52:38] (step=0038850) Train Loss: 6.3616, Train Steps/Sec: 0.91 + 85%|████████████████████████████████████████████████████████████████████████████████████████▉ | 38874/45917 [14:30:54<2:09:24, 1.10s/it][2025-04-24 07:53:05] (step=0038875) Train Loss: 6.3567, Train Steps/Sec: 0.90 + 85%|████████████████████████████████████████████████████████████████████████████████████████▉ | 38899/45917 [14:31:22<2:09:05, 1.10s/it][2025-04-24 07:53:33] (step=0038900) Train Loss: 6.3689, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████ | 38924/45917 [14:31:50<2:11:06, 1.12s/it][2025-04-24 07:54:01] (step=0038925) Train Loss: 6.3386, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████ | 38949/45917 [14:32:17<2:08:08, 1.10s/it][2025-04-24 07:54:29] (step=0038950) Train Loss: 6.3600, Train Steps/Sec: 0.91 + 85%|█████████████████████████████████████████████████████████████████████████████████████████ | 38974/45917 [14:32:45<2:08:19, 1.11s/it][2025-04-24 07:54:56] (step=0038975) Train Loss: 6.3608, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 38999/45917 [14:33:13<2:06:56, 1.10s/it][2025-04-24 07:55:24] (step=0039000) Train Loss: 6.3728, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 39024/45917 [14:33:41<2:09:20, 1.13s/it][2025-04-24 07:55:52] (step=0039025) Train Loss: 6.3372, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 39049/45917 [14:34:08<2:07:28, 1.11s/it][2025-04-24 07:56:20] (step=0039050) Train Loss: 6.3930, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 39074/45917 [14:34:36<2:05:19, 1.10s/it][2025-04-24 07:56:47] (step=0039075) Train Loss: 6.3405, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 39099/45917 [14:35:04<2:03:41, 1.09s/it][2025-04-24 07:57:15] (step=0039100) Train Loss: 6.3590, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 39124/45917 [14:35:32<2:07:21, 1.12s/it][2025-04-24 07:57:43] (step=0039125) Train Loss: 6.3975, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 39149/45917 [14:35:59<2:04:59, 1.11s/it][2025-04-24 07:58:10] (step=0039150) Train Loss: 6.3850, Train Steps/Sec: 0.91 + 85%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 39174/45917 [14:36:27<2:04:03, 1.10s/it][2025-04-24 07:58:38] (step=0039175) Train Loss: 6.2909, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 39199/45917 [14:36:55<2:02:31, 1.09s/it][2025-04-24 07:59:06] (step=0039200) Train Loss: 6.3605, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 39224/45917 [14:37:22<2:05:56, 1.13s/it][2025-04-24 07:59:34] (step=0039225) Train Loss: 6.3977, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 39249/45917 [14:37:50<2:03:10, 1.11s/it][2025-04-24 08:00:01] (step=0039250) Train Loss: 6.3235, Train Steps/Sec: 0.91 + 86%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 39274/45917 [14:38:18<2:01:26, 1.10s/it][2025-04-24 08:00:29] (step=0039275) Train Loss: 6.3644, Train Steps/Sec: 0.90 + 86%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 39299/45917 [14:38:45<2:00:59, 1.10s/it][2025-04-24 08:00:57] (step=0039300) Train Loss: 6.3454, Train Steps/Sec: 0.90 + 86%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 39324/45917 [14:39:13<2:03:28, 1.12s/it][2025-04-24 08:01:24] (step=0039325) Train Loss: 6.3640, Train Steps/Sec: 0.90 + 86%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 39349/45917 [14:39:41<2:02:29, 1.12s/it][2025-04-24 08:01:52] (step=0039350) Train Loss: 6.3139, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████ | 39374/45917 [14:40:09<1:59:49, 1.10s/it][2025-04-24 08:02:20] (step=0039375) Train Loss: 6.3475, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████ | 39399/45917 [14:40:36<1:59:19, 1.10s/it][2025-04-24 08:02:48] (step=0039400) Train Loss: 6.3771, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 39424/45917 [14:41:04<2:01:25, 1.12s/it][2025-04-24 08:03:15] (step=0039425) Train Loss: 6.3446, Train Steps/Sec: 0.91 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 39449/45917 [14:41:32<1:59:31, 1.11s/it][2025-04-24 08:03:43] (step=0039450) Train Loss: 6.3681, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 39474/45917 [14:41:59<1:58:26, 1.10s/it][2025-04-24 08:04:10] (step=0039475) Train Loss: 6.3590, Train Steps/Sec: 0.91 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 39499/45917 [14:42:27<1:57:09, 1.10s/it][2025-04-24 08:04:38] (step=0039500) Train Loss: 6.2992, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 39524/45917 [14:42:55<1:59:19, 1.12s/it][2025-04-24 08:05:06] (step=0039525) Train Loss: 6.3760, Train Steps/Sec: 0.91 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 39549/45917 [14:43:22<1:58:03, 1.11s/it][2025-04-24 08:05:33] (step=0039550) Train Loss: 6.3266, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 39574/45917 [14:43:50<1:56:50, 1.11s/it][2025-04-24 08:06:01] (step=0039575) Train Loss: 6.3760, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 39599/45917 [14:44:18<1:55:08, 1.09s/it][2025-04-24 08:06:29] (step=0039600) Train Loss: 6.3832, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 39624/45917 [14:44:45<1:58:35, 1.13s/it][2025-04-24 08:06:57] (step=0039625) Train Loss: 6.3614, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 39649/45917 [14:45:13<1:55:28, 1.11s/it][2025-04-24 08:07:24] (step=0039650) Train Loss: 6.3509, Train Steps/Sec: 0.91 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 39674/45917 [14:45:41<1:54:56, 1.10s/it][2025-04-24 08:07:52] (step=0039675) Train Loss: 6.3557, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 39699/45917 [14:46:08<1:53:27, 1.09s/it][2025-04-24 08:08:20] (step=0039700) Train Loss: 6.3526, Train Steps/Sec: 0.90 + 87%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 39724/45917 [14:46:36<1:56:13, 1.13s/it][2025-04-24 08:08:47] (step=0039725) Train Loss: 6.3636, Train Steps/Sec: 0.90 + 87%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 39749/45917 [14:47:04<1:54:34, 1.11s/it][2025-04-24 08:09:15] (step=0039750) Train Loss: 6.3298, Train Steps/Sec: 0.90 + 87%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 39774/45917 [14:47:32<1:53:05, 1.10s/it][2025-04-24 08:09:43] (step=0039775) Train Loss: 6.3415, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████ | 39799/45917 [14:48:00<1:52:11, 1.10s/it][2025-04-24 08:10:11] (step=0039800) Train Loss: 6.3363, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████ | 39824/45917 [14:48:27<1:54:16, 1.13s/it][2025-04-24 08:10:39] (step=0039825) Train Loss: 6.3475, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████ | 39849/45917 [14:48:55<1:51:50, 1.11s/it][2025-04-24 08:11:06] (step=0039850) Train Loss: 6.3994, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 39874/45917 [14:49:23<1:50:50, 1.10s/it][2025-04-24 08:11:34] (step=0039875) Train Loss: 6.3479, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 39899/45917 [14:49:50<1:49:54, 1.10s/it][2025-04-24 08:12:02] (step=0039900) Train Loss: 6.3512, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 39924/45917 [14:50:18<1:52:53, 1.13s/it][2025-04-24 08:12:29] (step=0039925) Train Loss: 6.4063, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 39949/45917 [14:50:46<1:50:15, 1.11s/it][2025-04-24 08:12:57] (step=0039950) Train Loss: 6.3267, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 39974/45917 [14:51:14<1:49:05, 1.10s/it][2025-04-24 08:13:25] (step=0039975) Train Loss: 6.3490, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 39999/45917 [14:51:41<1:48:31, 1.10s/it][2025-04-24 08:13:53] (step=0040000) Train Loss: 6.3519, Train Steps/Sec: 0.90 +[2025-04-24 08:13:53] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 08:13:53] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 08:13:53] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 59.29s/it] +[2025-04-24 08:18:41] Finish Eval in 40000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 58.90s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 08:19:04] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0040000.pt +[2025-04-24 08:19:06] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0038000.pt + 87%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 40024/45917 [14:57:23<1:52:44, 1.15s/it][2025-04-24 08:19:34] (step=0040025) Train Loss: 6.3527, Train Steps/Sec: 0.07 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 40049/45917 [14:57:50<1:48:17, 1.11s/it][2025-04-24 08:20:02] (step=0040050) Train Loss: 6.3614, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 40074/45917 [14:58:18<1:48:46, 1.12s/it][2025-04-24 08:20:29] (step=0040075) Train Loss: 6.3399, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 40099/45917 [14:58:46<1:45:40, 1.09s/it][2025-04-24 08:20:57] (step=0040100) Train Loss: 6.3684, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 40124/45917 [14:59:14<1:48:14, 1.12s/it][2025-04-24 08:21:25] (step=0040125) Train Loss: 6.3262, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 40149/45917 [14:59:41<1:45:51, 1.10s/it][2025-04-24 08:21:52] (step=0040150) Train Loss: 6.3710, Train Steps/Sec: 0.91 + 87%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 40174/45917 [15:00:09<1:45:23, 1.10s/it][2025-04-24 08:22:20] (step=0040175) Train Loss: 6.3971, Train Steps/Sec: 0.90 + 88%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 40199/45917 [15:00:37<1:44:41, 1.10s/it][2025-04-24 08:22:48] (step=0040200) Train Loss: 6.3687, Train Steps/Sec: 0.90 + 88%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 40224/45917 [15:01:05<1:47:04, 1.13s/it][2025-04-24 08:23:16] (step=0040225) Train Loss: 6.3787, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████ | 40249/45917 [15:01:32<1:44:54, 1.11s/it][2025-04-24 08:23:43] (step=0040250) Train Loss: 6.4060, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████ | 40274/45917 [15:02:00<1:43:52, 1.10s/it][2025-04-24 08:24:11] (step=0040275) Train Loss: 6.3659, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 40299/45917 [15:02:28<1:43:32, 1.11s/it][2025-04-24 08:24:39] (step=0040300) Train Loss: 6.3456, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 40324/45917 [15:02:55<1:45:58, 1.14s/it][2025-04-24 08:25:07] (step=0040325) Train Loss: 6.3782, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 40349/45917 [15:03:23<1:43:39, 1.12s/it][2025-04-24 08:25:34] (step=0040350) Train Loss: 6.4015, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 40374/45917 [15:03:51<1:41:32, 1.10s/it][2025-04-24 08:26:02] (step=0040375) Train Loss: 6.3703, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 40399/45917 [15:04:19<1:41:33, 1.10s/it][2025-04-24 08:26:30] (step=0040400) Train Loss: 6.3157, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 40424/45917 [15:04:46<1:42:42, 1.12s/it][2025-04-24 08:26:57] (step=0040425) Train Loss: 6.2867, Train Steps/Sec: 0.91 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 40449/45917 [15:05:14<1:40:53, 1.11s/it][2025-04-24 08:27:25] (step=0040450) Train Loss: 6.3723, Train Steps/Sec: 0.91 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 40474/45917 [15:05:42<1:40:55, 1.11s/it][2025-04-24 08:27:53] (step=0040475) Train Loss: 6.3501, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 40499/45917 [15:06:10<1:39:49, 1.11s/it][2025-04-24 08:28:21] (step=0040500) Train Loss: 6.3742, Train Steps/Sec: 0.89 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 40524/45917 [15:06:37<1:41:14, 1.13s/it][2025-04-24 08:28:49] (step=0040525) Train Loss: 6.3618, Train Steps/Sec: 0.91 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 40549/45917 [15:07:05<1:38:27, 1.10s/it][2025-04-24 08:29:16] (step=0040550) Train Loss: 6.3490, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 40574/45917 [15:07:33<1:38:27, 1.11s/it][2025-04-24 08:29:44] (step=0040575) Train Loss: 6.3784, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 40599/45917 [15:08:00<1:36:07, 1.08s/it][2025-04-24 08:30:12] (step=0040600) Train Loss: 6.3312, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 40624/45917 [15:08:28<1:40:23, 1.14s/it][2025-04-24 08:30:39] (step=0040625) Train Loss: 6.3343, Train Steps/Sec: 0.90 + 89%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 40649/45917 [15:08:56<1:37:27, 1.11s/it][2025-04-24 08:31:07] (step=0040650) Train Loss: 6.3457, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████ | 40674/45917 [15:09:24<1:36:27, 1.10s/it][2025-04-24 08:31:35] (step=0040675) Train Loss: 6.3314, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████ | 40699/45917 [15:09:51<1:35:12, 1.09s/it][2025-04-24 08:32:03] (step=0040700) Train Loss: 6.3997, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████ | 40724/45917 [15:10:19<1:37:32, 1.13s/it][2025-04-24 08:32:30] (step=0040725) Train Loss: 6.3443, Train Steps/Sec: 0.91 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 40749/45917 [15:10:47<1:35:18, 1.11s/it][2025-04-24 08:32:58] (step=0040750) Train Loss: 6.4255, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 40774/45917 [15:11:15<1:34:19, 1.10s/it][2025-04-24 08:33:26] (step=0040775) Train Loss: 6.4078, Train Steps/Sec: 0.91 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 40799/45917 [15:11:42<1:33:45, 1.10s/it][2025-04-24 08:33:54] (step=0040800) Train Loss: 6.3265, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 40824/45917 [15:12:10<1:34:52, 1.12s/it][2025-04-24 08:34:21] (step=0040825) Train Loss: 6.3271, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 40849/45917 [15:12:38<1:34:46, 1.12s/it][2025-04-24 08:34:49] (step=0040850) Train Loss: 6.3814, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 40874/45917 [15:13:06<1:33:28, 1.11s/it][2025-04-24 08:35:17] (step=0040875) Train Loss: 6.4021, Train Steps/Sec: 0.89 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 40899/45917 [15:13:34<1:31:54, 1.10s/it][2025-04-24 08:35:45] (step=0040900) Train Loss: 6.3105, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 40924/45917 [15:14:02<1:33:38, 1.13s/it][2025-04-24 08:36:13] (step=0040925) Train Loss: 6.3584, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 40949/45917 [15:14:29<1:32:06, 1.11s/it][2025-04-24 08:36:40] (step=0040950) Train Loss: 6.3323, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 40974/45917 [15:14:57<1:30:59, 1.10s/it][2025-04-24 08:37:08] (step=0040975) Train Loss: 6.3226, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 40999/45917 [15:15:25<1:29:26, 1.09s/it][2025-04-24 08:37:36] (step=0041000) Train Loss: 6.3495, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 41024/45917 [15:15:52<1:31:48, 1.13s/it][2025-04-24 08:38:04] (step=0041025) Train Loss: 6.3615, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 41049/45917 [15:16:20<1:29:51, 1.11s/it][2025-04-24 08:38:31] (step=0041050) Train Loss: 6.3865, Train Steps/Sec: 0.90 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 41074/45917 [15:16:48<1:30:15, 1.12s/it][2025-04-24 08:38:59] (step=0041075) Train Loss: 6.3675, Train Steps/Sec: 0.90 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 41099/45917 [15:17:16<1:28:27, 1.10s/it][2025-04-24 08:39:27] (step=0041100) Train Loss: 6.3337, Train Steps/Sec: 0.90 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████ | 41124/45917 [15:17:43<1:29:20, 1.12s/it][2025-04-24 08:39:55] (step=0041125) Train Loss: 6.3613, Train Steps/Sec: 0.91 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████ | 41149/45917 [15:18:11<1:27:58, 1.11s/it][2025-04-24 08:40:22] (step=0041150) Train Loss: 6.3019, Train Steps/Sec: 0.90 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 41174/45917 [15:18:39<1:28:15, 1.12s/it][2025-04-24 08:40:50] (step=0041175) Train Loss: 6.3347, Train Steps/Sec: 0.90 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 41199/45917 [15:19:07<1:25:59, 1.09s/it][2025-04-24 08:41:18] (step=0041200) Train Loss: 6.3535, Train Steps/Sec: 0.89 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 41224/45917 [15:19:34<1:27:30, 1.12s/it][2025-04-24 08:41:46] (step=0041225) Train Loss: 6.3203, Train Steps/Sec: 0.91 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 41249/45917 [15:20:02<1:26:04, 1.11s/it][2025-04-24 08:42:13] (step=0041250) Train Loss: 6.3422, Train Steps/Sec: 0.90 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 41274/45917 [15:20:30<1:24:57, 1.10s/it][2025-04-24 08:42:41] (step=0041275) Train Loss: 6.3435, Train Steps/Sec: 0.91 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 41299/45917 [15:20:57<1:24:49, 1.10s/it][2025-04-24 08:43:09] (step=0041300) Train Loss: 6.4256, Train Steps/Sec: 0.90 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 41324/45917 [15:21:25<1:25:42, 1.12s/it][2025-04-24 08:43:36] (step=0041325) Train Loss: 6.3693, Train Steps/Sec: 0.90 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 41349/45917 [15:21:53<1:24:22, 1.11s/it][2025-04-24 08:44:04] (step=0041350) Train Loss: 6.3411, Train Steps/Sec: 0.90 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 41374/45917 [15:22:32<1:26:27, 1.14s/it][2025-04-24 08:44:43] (step=0041375) Train Loss: 6.3846, Train Steps/Sec: 0.65 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 41399/45917 [15:22:59<1:21:41, 1.08s/it][2025-04-24 08:45:10] (step=0041400) Train Loss: 6.3200, Train Steps/Sec: 0.90 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 41424/45917 [15:23:27<1:25:15, 1.14s/it][2025-04-24 08:45:38] (step=0041425) Train Loss: 6.3471, Train Steps/Sec: 0.91 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 41449/45917 [15:23:55<1:22:25, 1.11s/it][2025-04-24 08:46:06] (step=0041450) Train Loss: 6.3783, Train Steps/Sec: 0.90 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 41474/45917 [15:24:22<1:21:45, 1.10s/it][2025-04-24 08:46:34] (step=0041475) Train Loss: 6.3361, Train Steps/Sec: 0.90 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 41499/45917 [15:24:50<1:20:39, 1.10s/it][2025-04-24 08:47:02] (step=0041500) Train Loss: 6.3603, Train Steps/Sec: 0.89 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 41524/45917 [15:25:18<1:22:49, 1.13s/it][2025-04-24 08:47:35] (step=0041525) Train Loss: 6.3639, Train Steps/Sec: 0.74 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████ | 41549/45917 [15:25:52<1:20:46, 1.11s/it][2025-04-24 08:48:03] (step=0041550) Train Loss: 6.3655, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████ | 41574/45917 [15:26:20<1:20:05, 1.11s/it][2025-04-24 08:48:31] (step=0041575) Train Loss: 6.3449, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 41599/45917 [15:26:54<1:35:14, 1.32s/it][2025-04-24 08:49:05] (step=0041600) Train Loss: 6.3779, Train Steps/Sec: 0.73 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 41624/45917 [15:27:28<1:36:04, 1.34s/it][2025-04-24 08:49:39] (step=0041625) Train Loss: 6.3475, Train Steps/Sec: 0.74 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 41649/45917 [15:27:55<1:19:02, 1.11s/it][2025-04-24 08:50:07] (step=0041650) Train Loss: 6.3657, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 41674/45917 [15:28:23<1:17:33, 1.10s/it][2025-04-24 08:50:34] (step=0041675) Train Loss: 6.3518, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 41699/45917 [15:28:51<1:16:57, 1.09s/it][2025-04-24 08:51:02] (step=0041700) Train Loss: 6.3903, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 41724/45917 [15:29:19<1:18:41, 1.13s/it][2025-04-24 08:51:30] (step=0041725) Train Loss: 6.3501, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 41749/45917 [15:29:46<1:17:08, 1.11s/it][2025-04-24 08:51:58] (step=0041750) Train Loss: 6.3899, Train Steps/Sec: 0.91 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 41774/45917 [15:30:14<1:16:06, 1.10s/it][2025-04-24 08:52:25] (step=0041775) Train Loss: 6.3595, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 41799/45917 [15:30:42<1:14:48, 1.09s/it][2025-04-24 08:52:53] (step=0041800) Train Loss: 6.3512, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 41824/45917 [15:31:10<1:16:49, 1.13s/it][2025-04-24 08:53:21] (step=0041825) Train Loss: 6.3168, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 41849/45917 [15:31:37<1:15:50, 1.12s/it][2025-04-24 08:53:49] (step=0041850) Train Loss: 6.3305, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 41874/45917 [15:32:05<1:14:41, 1.11s/it][2025-04-24 08:54:16] (step=0041875) Train Loss: 6.3636, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 41899/45917 [15:32:33<1:13:41, 1.10s/it][2025-04-24 08:54:44] (step=0041900) Train Loss: 6.4077, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 41924/45917 [15:33:01<1:14:35, 1.12s/it][2025-04-24 08:55:12] (step=0041925) Train Loss: 6.3446, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 41949/45917 [15:33:28<1:13:36, 1.11s/it][2025-04-24 08:55:39] (step=0041950) Train Loss: 6.3238, Train Steps/Sec: 0.90 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 41974/45917 [15:33:56<1:12:13, 1.10s/it][2025-04-24 08:56:07] (step=0041975) Train Loss: 6.3677, Train Steps/Sec: 0.90 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████ | 41999/45917 [15:34:24<1:12:11, 1.11s/it][2025-04-24 08:56:35] (step=0042000) Train Loss: 6.3243, Train Steps/Sec: 0.90 +[2025-04-24 08:56:35] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 08:56:35] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 08:56:35] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.65s/it] +[2025-04-24 09:01:22] Finish Eval in 42000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.20s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 09:01:44] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0042000.pt +[2025-04-24 09:01:46] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0040000.pt + 92%|████████████████████████████████████████████████████████████████████████████████████████████████ | 42024/45917 [15:40:03<1:13:47, 1.14s/it][2025-04-24 09:02:14] (step=0042025) Train Loss: 6.4012, Train Steps/Sec: 0.07 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42049/45917 [15:40:31<1:11:34, 1.11s/it][2025-04-24 09:02:42] (step=0042050) Train Loss: 6.3805, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42074/45917 [15:40:58<1:11:20, 1.11s/it][2025-04-24 09:03:10] (step=0042075) Train Loss: 6.4211, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 42099/45917 [15:41:26<1:09:46, 1.10s/it][2025-04-24 09:03:37] (step=0042100) Train Loss: 6.3583, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 42124/45917 [15:41:54<1:11:39, 1.13s/it][2025-04-24 09:04:05] (step=0042125) Train Loss: 6.3955, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 42149/45917 [15:42:22<1:09:50, 1.11s/it][2025-04-24 09:04:33] (step=0042150) Train Loss: 6.3862, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 42174/45917 [15:42:49<1:08:40, 1.10s/it][2025-04-24 09:05:01] (step=0042175) Train Loss: 6.3665, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 42199/45917 [15:43:17<1:08:10, 1.10s/it][2025-04-24 09:05:28] (step=0042200) Train Loss: 6.3355, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42224/45917 [15:43:45<1:09:32, 1.13s/it][2025-04-24 09:05:56] (step=0042225) Train Loss: 6.3596, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42249/45917 [15:44:12<1:07:35, 1.11s/it][2025-04-24 09:06:24] (step=0042250) Train Loss: 6.2946, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 42274/45917 [15:44:40<1:07:27, 1.11s/it][2025-04-24 09:06:51] (step=0042275) Train Loss: 6.3548, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 42299/45917 [15:45:08<1:06:17, 1.10s/it][2025-04-24 09:07:19] (step=0042300) Train Loss: 6.3103, Train Steps/Sec: 0.89 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 42324/45917 [15:45:36<1:07:42, 1.13s/it][2025-04-24 09:07:47] (step=0042325) Train Loss: 6.3384, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 42349/45917 [15:46:04<1:07:25, 1.13s/it][2025-04-24 09:08:15] (step=0042350) Train Loss: 6.3444, Train Steps/Sec: 0.89 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 42374/45917 [15:46:32<1:05:01, 1.10s/it][2025-04-24 09:08:43] (step=0042375) Train Loss: 6.3754, Train Steps/Sec: 0.90 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 42399/45917 [15:46:59<1:03:55, 1.09s/it][2025-04-24 09:09:11] (step=0042400) Train Loss: 6.3435, Train Steps/Sec: 0.90 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 42424/45917 [15:47:27<1:05:26, 1.12s/it][2025-04-24 09:09:38] (step=0042425) Train Loss: 6.3216, Train Steps/Sec: 0.91 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 42449/45917 [15:47:55<1:04:05, 1.11s/it][2025-04-24 09:10:06] (step=0042450) Train Loss: 6.3346, Train Steps/Sec: 0.91 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42474/45917 [15:48:22<1:03:01, 1.10s/it][2025-04-24 09:10:33] (step=0042475) Train Loss: 6.3321, Train Steps/Sec: 0.91 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42499/45917 [15:48:50<1:02:51, 1.10s/it][2025-04-24 09:11:01] (step=0042500) Train Loss: 6.4028, Train Steps/Sec: 0.90 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42524/45917 [15:49:18<1:03:37, 1.13s/it][2025-04-24 09:11:29] (step=0042525) Train Loss: 6.3580, Train Steps/Sec: 0.91 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 42549/45917 [15:49:45<1:02:42, 1.12s/it][2025-04-24 09:11:57] (step=0042550) Train Loss: 6.3415, Train Steps/Sec: 0.90 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 42574/45917 [15:50:13<1:01:00, 1.09s/it][2025-04-24 09:12:24] (step=0042575) Train Loss: 6.3110, Train Steps/Sec: 0.91 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 42599/45917 [15:50:41<1:01:09, 1.11s/it][2025-04-24 09:12:52] (step=0042600) Train Loss: 6.3850, Train Steps/Sec: 0.90 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 42624/45917 [15:51:08<1:01:51, 1.13s/it][2025-04-24 09:13:20] (step=0042625) Train Loss: 6.3590, Train Steps/Sec: 0.91 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42649/45917 [15:51:36<1:00:11, 1.11s/it][2025-04-24 09:13:47] (step=0042650) Train Loss: 6.2832, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 42674/45917 [15:52:04<59:18, 1.10s/it][2025-04-24 09:14:15] (step=0042675) Train Loss: 6.3348, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42699/45917 [15:52:31<57:57, 1.08s/it][2025-04-24 09:14:43] (step=0042700) Train Loss: 6.3692, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42724/45917 [15:52:59<59:32, 1.12s/it][2025-04-24 09:15:10] (step=0042725) Train Loss: 6.3082, Train Steps/Sec: 0.91 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42749/45917 [15:53:27<58:40, 1.11s/it][2025-04-24 09:15:38] (step=0042750) Train Loss: 6.3371, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 42774/45917 [15:53:54<58:06, 1.11s/it][2025-04-24 09:16:06] (step=0042775) Train Loss: 6.3749, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 42799/45917 [15:54:22<56:49, 1.09s/it][2025-04-24 09:16:33] (step=0042800) Train Loss: 6.3396, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 42824/45917 [15:54:50<57:57, 1.12s/it][2025-04-24 09:17:01] (step=0042825) Train Loss: 6.3161, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 42849/45917 [15:55:18<57:08, 1.12s/it][2025-04-24 09:17:29] (step=0042850) Train Loss: 6.4434, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 42874/45917 [15:55:45<55:31, 1.09s/it][2025-04-24 09:17:56] (step=0042875) Train Loss: 6.3474, Train Steps/Sec: 0.91 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 42899/45917 [15:56:13<54:18, 1.08s/it][2025-04-24 09:18:24] (step=0042900) Train Loss: 6.3249, Train Steps/Sec: 0.90 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 42924/45917 [15:56:41<55:56, 1.12s/it][2025-04-24 09:18:52] (step=0042925) Train Loss: 6.3567, Train Steps/Sec: 0.91 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 42949/45917 [15:57:08<55:36, 1.12s/it][2025-04-24 09:19:19] (step=0042950) Train Loss: 6.3344, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42974/45917 [15:57:36<53:40, 1.09s/it][2025-04-24 09:19:47] (step=0042975) Train Loss: 6.3625, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42999/45917 [15:58:04<53:31, 1.10s/it][2025-04-24 09:20:15] (step=0043000) Train Loss: 6.3281, Train Steps/Sec: 0.89 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43024/45917 [15:58:32<54:07, 1.12s/it][2025-04-24 09:20:43] (step=0043025) Train Loss: 6.3611, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43049/45917 [15:58:59<52:48, 1.10s/it][2025-04-24 09:21:11] (step=0043050) Train Loss: 6.3440, Train Steps/Sec: 0.91 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43074/45917 [15:59:27<52:03, 1.10s/it][2025-04-24 09:21:38] (step=0043075) Train Loss: 6.3290, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 43099/45917 [15:59:55<51:58, 1.11s/it][2025-04-24 09:22:06] (step=0043100) Train Loss: 6.3209, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 43124/45917 [16:00:23<52:51, 1.14s/it][2025-04-24 09:22:34] (step=0043125) Train Loss: 6.3696, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 43149/45917 [16:00:50<51:07, 1.11s/it][2025-04-24 09:23:02] (step=0043150) Train Loss: 6.2905, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 43174/45917 [16:01:18<49:50, 1.09s/it][2025-04-24 09:23:29] (step=0043175) Train Loss: 6.3593, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 43199/45917 [16:01:46<49:35, 1.09s/it][2025-04-24 09:23:57] (step=0043200) Train Loss: 6.3659, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 43224/45917 [16:02:14<50:28, 1.12s/it][2025-04-24 09:24:25] (step=0043225) Train Loss: 6.3327, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 43249/45917 [16:02:42<49:41, 1.12s/it][2025-04-24 09:24:53] (step=0043250) Train Loss: 6.3690, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 43274/45917 [16:03:10<48:34, 1.10s/it][2025-04-24 09:25:21] (step=0043275) Train Loss: 6.3864, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43299/45917 [16:03:37<47:53, 1.10s/it][2025-04-24 09:25:49] (step=0043300) Train Loss: 6.3281, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43324/45917 [16:04:06<48:54, 1.13s/it][2025-04-24 09:26:17] (step=0043325) Train Loss: 6.3755, Train Steps/Sec: 0.90 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 43349/45917 [16:04:33<47:31, 1.11s/it][2025-04-24 09:26:45] (step=0043350) Train Loss: 6.2994, Train Steps/Sec: 0.90 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 43374/45917 [16:05:01<46:51, 1.11s/it][2025-04-24 09:27:12] (step=0043375) Train Loss: 6.3575, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 43399/45917 [16:05:29<46:26, 1.11s/it][2025-04-24 09:27:40] (step=0043400) Train Loss: 6.3956, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 43424/45917 [16:05:57<46:51, 1.13s/it][2025-04-24 09:28:08] (step=0043425) Train Loss: 6.3428, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 43449/45917 [16:06:33<47:04, 1.14s/it][2025-04-24 09:28:44] (step=0043450) Train Loss: 6.3291, Train Steps/Sec: 0.69 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43474/45917 [16:07:08<46:33, 1.14s/it][2025-04-24 09:29:19] (step=0043475) Train Loss: 6.4169, Train Steps/Sec: 0.71 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43499/45917 [16:07:36<44:08, 1.10s/it][2025-04-24 09:29:47] (step=0043500) Train Loss: 6.3541, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 43524/45917 [16:08:04<44:57, 1.13s/it][2025-04-24 09:30:15] (step=0043525) Train Loss: 6.4008, Train Steps/Sec: 0.91 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 43549/45917 [16:08:31<43:56, 1.11s/it][2025-04-24 09:30:42] (step=0043550) Train Loss: 6.3591, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 43574/45917 [16:08:59<42:57, 1.10s/it][2025-04-24 09:31:10] (step=0043575) Train Loss: 6.3454, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 43599/45917 [16:09:27<42:12, 1.09s/it][2025-04-24 09:31:38] (step=0043600) Train Loss: 6.3254, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 43624/45917 [16:09:54<42:46, 1.12s/it][2025-04-24 09:32:06] (step=0043625) Train Loss: 6.3115, Train Steps/Sec: 0.91 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 43649/45917 [16:10:22<42:12, 1.12s/it][2025-04-24 09:32:33] (step=0043650) Train Loss: 6.3494, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 43674/45917 [16:10:50<40:56, 1.10s/it][2025-04-24 09:33:01] (step=0043675) Train Loss: 6.3885, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 43699/45917 [16:11:17<40:17, 1.09s/it][2025-04-24 09:33:29] (step=0043700) Train Loss: 6.3632, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43724/45917 [16:11:45<41:06, 1.12s/it][2025-04-24 09:33:56] (step=0043725) Train Loss: 6.3100, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43749/45917 [16:12:13<40:13, 1.11s/it][2025-04-24 09:34:24] (step=0043750) Train Loss: 6.3165, Train Steps/Sec: 0.90 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 43774/45917 [16:12:41<39:32, 1.11s/it][2025-04-24 09:34:52] (step=0043775) Train Loss: 6.3504, Train Steps/Sec: 0.90 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 43799/45917 [16:13:08<38:45, 1.10s/it][2025-04-24 09:35:20] (step=0043800) Train Loss: 6.3323, Train Steps/Sec: 0.90 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 43824/45917 [16:13:36<39:06, 1.12s/it][2025-04-24 09:35:47] (step=0043825) Train Loss: 6.3217, Train Steps/Sec: 0.91 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 43849/45917 [16:14:04<38:08, 1.11s/it][2025-04-24 09:36:15] (step=0043850) Train Loss: 6.3529, Train Steps/Sec: 0.91 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 43874/45917 [16:14:31<37:37, 1.11s/it][2025-04-24 09:36:43] (step=0043875) Train Loss: 6.3098, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43899/45917 [16:14:59<36:54, 1.10s/it][2025-04-24 09:37:10] (step=0043900) Train Loss: 6.3675, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43924/45917 [16:15:27<37:26, 1.13s/it][2025-04-24 09:37:38] (step=0043925) Train Loss: 6.3710, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 43949/45917 [16:15:55<36:27, 1.11s/it][2025-04-24 09:38:06] (step=0043950) Train Loss: 6.3725, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 43974/45917 [16:16:22<35:39, 1.10s/it][2025-04-24 09:38:33] (step=0043975) Train Loss: 6.3369, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 43999/45917 [16:16:50<34:56, 1.09s/it][2025-04-24 09:39:01] (step=0044000) Train Loss: 6.3471, Train Steps/Sec: 0.90 +[2025-04-24 09:39:01] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 09:39:01] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 09:39:01] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.64s/it] +[2025-04-24 09:43:49] Finish Eval in 44000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.19s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 09:44:12] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0044000.pt +[2025-04-24 09:44:14] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0042000.pt + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 44024/45917 [16:22:31<35:53, 1.14s/it][2025-04-24 09:44:42] (step=0044025) Train Loss: 6.3830, Train Steps/Sec: 0.07 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 44049/45917 [16:22:58<34:30, 1.11s/it][2025-04-24 09:45:10] (step=0044050) Train Loss: 6.3829, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 44074/45917 [16:23:26<33:50, 1.10s/it][2025-04-24 09:45:37] (step=0044075) Train Loss: 6.3609, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44099/45917 [16:23:54<33:10, 1.09s/it][2025-04-24 09:46:05] (step=0044100) Train Loss: 6.3875, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44124/45917 [16:24:21<33:36, 1.12s/it][2025-04-24 09:46:33] (step=0044125) Train Loss: 6.3950, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 44149/45917 [16:24:49<32:46, 1.11s/it][2025-04-24 09:47:00] (step=0044150) Train Loss: 6.2873, Train Steps/Sec: 0.91 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 44174/45917 [16:25:17<31:58, 1.10s/it][2025-04-24 09:47:28] (step=0044175) Train Loss: 6.3772, Train Steps/Sec: 0.91 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 44199/45917 [16:25:44<31:33, 1.10s/it][2025-04-24 09:47:56] (step=0044200) Train Loss: 6.3264, Train Steps/Sec: 0.90 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 44224/45917 [16:26:13<31:47, 1.13s/it][2025-04-24 09:48:24] (step=0044225) Train Loss: 6.3864, Train Steps/Sec: 0.89 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 44249/45917 [16:26:40<30:53, 1.11s/it][2025-04-24 09:48:51] (step=0044250) Train Loss: 6.3640, Train Steps/Sec: 0.91 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 44274/45917 [16:27:08<30:06, 1.10s/it][2025-04-24 09:49:19] (step=0044275) Train Loss: 6.3770, Train Steps/Sec: 0.90 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 44299/45917 [16:27:36<29:38, 1.10s/it][2025-04-24 09:49:47] (step=0044300) Train Loss: 6.3408, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 44324/45917 [16:28:03<29:30, 1.11s/it][2025-04-24 09:50:14] (step=0044325) Train Loss: 6.3624, Train Steps/Sec: 0.91 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 44349/45917 [16:28:31<28:44, 1.10s/it][2025-04-24 09:50:42] (step=0044350) Train Loss: 6.3639, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 44374/45917 [16:28:58<28:05, 1.09s/it][2025-04-24 09:51:10] (step=0044375) Train Loss: 6.3376, Train Steps/Sec: 0.91 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 44399/45917 [16:29:26<27:36, 1.09s/it][2025-04-24 09:51:37] (step=0044400) Train Loss: 6.3690, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 44424/45917 [16:29:54<27:52, 1.12s/it][2025-04-24 09:52:05] (step=0044425) Train Loss: 6.3234, Train Steps/Sec: 0.91 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 44449/45917 [16:30:21<27:09, 1.11s/it][2025-04-24 09:52:33] (step=0044450) Train Loss: 6.3608, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 44474/45917 [16:30:49<26:34, 1.11s/it][2025-04-24 09:53:00] (step=0044475) Train Loss: 6.3671, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 44499/45917 [16:31:17<26:04, 1.10s/it][2025-04-24 09:53:28] (step=0044500) Train Loss: 6.3428, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44524/45917 [16:31:45<25:58, 1.12s/it][2025-04-24 09:53:56] (step=0044525) Train Loss: 6.2929, Train Steps/Sec: 0.91 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44549/45917 [16:32:12<25:14, 1.11s/it][2025-04-24 09:54:23] (step=0044550) Train Loss: 6.3406, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44574/45917 [16:32:40<24:37, 1.10s/it][2025-04-24 09:54:51] (step=0044575) Train Loss: 6.3702, Train Steps/Sec: 0.91 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 44599/45917 [16:33:08<24:15, 1.10s/it][2025-04-24 09:55:19] (step=0044600) Train Loss: 6.3588, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 44624/45917 [16:33:35<24:12, 1.12s/it][2025-04-24 09:55:47] (step=0044625) Train Loss: 6.3283, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 44649/45917 [16:34:03<23:23, 1.11s/it][2025-04-24 09:56:14] (step=0044650) Train Loss: 6.3597, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 44674/45917 [16:34:31<22:52, 1.10s/it][2025-04-24 09:56:42] (step=0044675) Train Loss: 6.3201, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 44699/45917 [16:34:59<22:21, 1.10s/it][2025-04-24 09:57:10] (step=0044700) Train Loss: 6.3611, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 44724/45917 [16:35:26<22:30, 1.13s/it][2025-04-24 09:57:38] (step=0044725) Train Loss: 6.3471, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 44749/45917 [16:35:54<21:33, 1.11s/it][2025-04-24 09:58:05] (step=0044750) Train Loss: 6.3314, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 44774/45917 [16:36:22<21:02, 1.10s/it][2025-04-24 09:58:33] (step=0044775) Train Loss: 6.3319, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 44799/45917 [16:36:49<20:22, 1.09s/it][2025-04-24 09:59:01] (step=0044800) Train Loss: 6.3636, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 44824/45917 [16:37:17<20:28, 1.12s/it][2025-04-24 09:59:28] (step=0044825) Train Loss: 6.3487, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 44849/45917 [16:37:45<19:55, 1.12s/it][2025-04-24 09:59:56] (step=0044850) Train Loss: 6.3567, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 44874/45917 [16:38:13<19:11, 1.10s/it][2025-04-24 10:00:24] (step=0044875) Train Loss: 6.3529, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 44899/45917 [16:38:40<18:35, 1.10s/it][2025-04-24 10:00:52] (step=0044900) Train Loss: 6.3244, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 44924/45917 [16:39:08<18:35, 1.12s/it][2025-04-24 10:01:19] (step=0044925) Train Loss: 6.3840, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 44949/45917 [16:39:36<18:00, 1.12s/it][2025-04-24 10:01:47] (step=0044950) Train Loss: 6.3561, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44974/45917 [16:40:04<17:25, 1.11s/it][2025-04-24 10:02:15] (step=0044975) Train Loss: 6.3597, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44999/45917 [16:40:31<16:43, 1.09s/it][2025-04-24 10:02:43] (step=0045000) Train Loss: 6.3965, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 45024/45917 [16:40:59<16:45, 1.13s/it][2025-04-24 10:03:10] (step=0045025) Train Loss: 6.3348, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 45049/45917 [16:41:27<15:56, 1.10s/it][2025-04-24 10:03:38] (step=0045050) Train Loss: 6.3268, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 45074/45917 [16:41:55<15:39, 1.11s/it][2025-04-24 10:04:06] (step=0045075) Train Loss: 6.3811, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 45099/45917 [16:42:22<14:52, 1.09s/it][2025-04-24 10:04:34] (step=0045100) Train Loss: 6.3674, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 45124/45917 [16:42:50<14:54, 1.13s/it][2025-04-24 10:05:01] (step=0045125) Train Loss: 6.3657, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 45149/45917 [16:43:18<14:12, 1.11s/it][2025-04-24 10:05:29] (step=0045150) Train Loss: 6.3090, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 45174/45917 [16:43:46<13:44, 1.11s/it][2025-04-24 10:05:57] (step=0045175) Train Loss: 6.3893, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 45199/45917 [16:44:13<13:06, 1.09s/it][2025-04-24 10:06:25] (step=0045200) Train Loss: 6.3358, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 45224/45917 [16:44:41<12:55, 1.12s/it][2025-04-24 10:06:52] (step=0045225) Train Loss: 6.3661, Train Steps/Sec: 0.90 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 45249/45917 [16:45:09<12:23, 1.11s/it][2025-04-24 10:07:20] (step=0045250) Train Loss: 6.3833, Train Steps/Sec: 0.90 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 45274/45917 [16:45:36<11:46, 1.10s/it][2025-04-24 10:07:48] (step=0045275) Train Loss: 6.2981, Train Steps/Sec: 0.91 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 45299/45917 [16:46:04<11:15, 1.09s/it][2025-04-24 10:08:16] (step=0045300) Train Loss: 6.3273, Train Steps/Sec: 0.90 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 45324/45917 [16:46:32<11:14, 1.14s/it][2025-04-24 10:08:43] (step=0045325) Train Loss: 6.4225, Train Steps/Sec: 0.90 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 45349/45917 [16:47:00<10:28, 1.11s/it][2025-04-24 10:09:11] (step=0045350) Train Loss: 6.3711, Train Steps/Sec: 0.90 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 45374/45917 [16:47:27<10:00, 1.11s/it][2025-04-24 10:09:39] (step=0045375) Train Loss: 6.3535, Train Steps/Sec: 0.90 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 45399/45917 [16:47:55<09:26, 1.09s/it][2025-04-24 10:10:06] (step=0045400) Train Loss: 6.3772, Train Steps/Sec: 0.90 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 45424/45917 [16:48:23<09:15, 1.13s/it][2025-04-24 10:10:34] (step=0045425) Train Loss: 6.3800, Train Steps/Sec: 0.91 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 45449/45917 [16:48:50<08:37, 1.11s/it][2025-04-24 10:11:02] (step=0045450) Train Loss: 6.3854, Train Steps/Sec: 0.90 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 45474/45917 [16:49:18<08:06, 1.10s/it][2025-04-24 10:11:29] (step=0045475) Train Loss: 6.3018, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 45499/45917 [16:49:46<07:40, 1.10s/it][2025-04-24 10:11:57] (step=0045500) Train Loss: 6.3530, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 45524/45917 [16:50:14<07:24, 1.13s/it][2025-04-24 10:12:25] (step=0045525) Train Loss: 6.3655, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 45549/45917 [16:50:41<06:52, 1.12s/it][2025-04-24 10:12:53] (step=0045550) Train Loss: 6.3810, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 45574/45917 [16:51:09<06:18, 1.10s/it][2025-04-24 10:13:20] (step=0045575) Train Loss: 6.3752, Train Steps/Sec: 0.91 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 45599/45917 [16:51:37<05:44, 1.08s/it][2025-04-24 10:13:48] (step=0045600) Train Loss: 6.3597, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 45624/45917 [16:52:04<05:27, 1.12s/it][2025-04-24 10:14:16] (step=0045625) Train Loss: 6.4101, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 45649/45917 [16:52:32<04:58, 1.11s/it][2025-04-24 10:14:43] (step=0045650) Train Loss: 6.3473, Train Steps/Sec: 0.91 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 45674/45917 [16:53:00<04:25, 1.09s/it][2025-04-24 10:15:11] (step=0045675) Train Loss: 6.3746, Train Steps/Sec: 0.91 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 45699/45917 [16:53:27<03:59, 1.10s/it][2025-04-24 10:15:39] (step=0045700) Train Loss: 6.3494, Train Steps/Sec: 0.90 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 45724/45917 [16:53:55<03:38, 1.13s/it][2025-04-24 10:16:06] (step=0045725) Train Loss: 6.3205, Train Steps/Sec: 0.90 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 45749/45917 [16:54:23<03:05, 1.10s/it][2025-04-24 10:16:34] (step=0045750) Train Loss: 6.3742, Train Steps/Sec: 0.90 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 45774/45917 [16:54:50<02:37, 1.10s/it][2025-04-24 10:17:02] (step=0045775) Train Loss: 6.3723, Train Steps/Sec: 0.90 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 45799/45917 [16:55:18<02:10, 1.10s/it][2025-04-24 10:17:29] (step=0045800) Train Loss: 6.3506, Train Steps/Sec: 0.90 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 45824/45917 [16:55:46<01:45, 1.13s/it][2025-04-24 10:17:57] (step=0045825) Train Loss: 6.3764, Train Steps/Sec: 0.90 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 45849/45917 [16:56:13<01:15, 1.10s/it][2025-04-24 10:18:25] (step=0045850) Train Loss: 6.3246, Train Steps/Sec: 0.91 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 45874/45917 [16:56:41<00:47, 1.09s/it][2025-04-24 10:18:52] (step=0045875) Train Loss: 6.3787, Train Steps/Sec: 0.91 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 45899/45917 [16:57:09<00:19, 1.10s/it][2025-04-24 10:19:20] (step=0045900) Train Loss: 6.3160, Train Steps/Sec: 0.90 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 45917/45917 [16:57:30<00:00, 1.33s/it] +[2025-04-24 10:19:41] Beginning epoch 1... + 0%| | 7/45917 [00:09<15:34:55, 1.22s/it][2025-04-24 10:19:51] (step=0045925) Train Loss: 6.2801, Train Steps/Sec: 0.79 + 0%| | 32/45917 [00:37<14:11:12, 1.11s/it][2025-04-24 10:20:19] (step=0045950) Train Loss: 6.3535, Train Steps/Sec: 0.90 + 0%|▏ | 57/45917 [01:05<14:07:50, 1.11s/it][2025-04-24 10:20:47] (step=0045975) Train Loss: 6.3566, Train Steps/Sec: 0.90 + 0%|▏ | 82/45917 [01:33<14:07:04, 1.11s/it][2025-04-24 10:21:15] (step=0046000) Train Loss: 6.3297, Train Steps/Sec: 0.90 +[2025-04-24 10:21:15] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 10:21:15] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 10:21:15] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 59.10s/it] +[2025-04-24 10:26:07] Finish Eval in 46000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.57s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 10:26:29] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0046000.pt +[2025-04-24 10:26:31] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0044000.pt + 0%|▎ | 107/45917 [07:17<14:33:20, 1.14s/it][2025-04-24 10:26:59] (step=0046025) Train Loss: 6.3272, Train Steps/Sec: 0.07 + 0%|▍ | 132/45917 [07:45<14:10:28, 1.11s/it][2025-04-24 10:27:27] (step=0046050) Train Loss: 6.2840, Train Steps/Sec: 0.90 + 0%|▍ | 157/45917 [08:12<14:11:02, 1.12s/it][2025-04-24 10:27:55] (step=0046075) Train Loss: 6.3496, Train Steps/Sec: 0.89 + 0%|▌ | 182/45917 [08:41<13:58:24, 1.10s/it][2025-04-24 10:28:23] (step=0046100) Train Loss: 6.2827, Train Steps/Sec: 0.88 + 0%|▌ | 207/45917 [09:09<14:21:50, 1.13s/it][2025-04-24 10:28:51] (step=0046125) Train Loss: 6.3419, Train Steps/Sec: 0.90 + 1%|▋ | 232/45917 [09:37<14:02:01, 1.11s/it][2025-04-24 10:29:19] (step=0046150) Train Loss: 6.3212, Train Steps/Sec: 0.90 + 1%|▊ | 257/45917 [10:04<14:11:00, 1.12s/it][2025-04-24 10:29:47] (step=0046175) Train Loss: 6.3141, Train Steps/Sec: 0.90 + 1%|▊ | 282/45917 [10:32<13:57:15, 1.10s/it][2025-04-24 10:30:15] (step=0046200) Train Loss: 6.3333, Train Steps/Sec: 0.89 + 1%|▉ | 307/45917 [11:00<14:15:50, 1.13s/it][2025-04-24 10:30:42] (step=0046225) Train Loss: 6.2786, Train Steps/Sec: 0.90 + 1%|▉ | 332/45917 [11:28<14:16:31, 1.13s/it][2025-04-24 10:31:10] (step=0046250) Train Loss: 6.3614, Train Steps/Sec: 0.89 + 1%|█ | 357/45917 [11:56<14:03:31, 1.11s/it][2025-04-24 10:31:38] (step=0046275) Train Loss: 6.3277, Train Steps/Sec: 0.90 + 1%|█▏ | 382/45917 [12:24<13:55:36, 1.10s/it][2025-04-24 10:32:06] (step=0046300) Train Loss: 6.3522, Train Steps/Sec: 0.90 + 1%|█▏ | 407/45917 [12:52<14:10:49, 1.12s/it][2025-04-24 10:32:34] (step=0046325) Train Loss: 6.3099, Train Steps/Sec: 0.90 + 1%|█▎ | 432/45917 [13:19<13:58:01, 1.11s/it][2025-04-24 10:33:02] (step=0046350) Train Loss: 6.3623, Train Steps/Sec: 0.90 + 1%|█▎ | 457/45917 [13:47<14:00:25, 1.11s/it][2025-04-24 10:33:29] (step=0046375) Train Loss: 6.2890, Train Steps/Sec: 0.90 + 1%|█▍ | 482/45917 [14:15<13:47:15, 1.09s/it][2025-04-24 10:33:57] (step=0046400) Train Loss: 6.3479, Train Steps/Sec: 0.90 + 1%|█▌ | 507/45917 [14:43<14:09:45, 1.12s/it][2025-04-24 10:34:25] (step=0046425) Train Loss: 6.3926, Train Steps/Sec: 0.90 + 1%|█▌ | 532/45917 [15:11<14:02:09, 1.11s/it][2025-04-24 10:34:53] (step=0046450) Train Loss: 6.3761, Train Steps/Sec: 0.90 + 1%|█▋ | 557/45917 [15:39<14:04:49, 1.12s/it][2025-04-24 10:35:21] (step=0046475) Train Loss: 6.3530, Train Steps/Sec: 0.90 + 1%|█▋ | 582/45917 [16:06<13:50:46, 1.10s/it][2025-04-24 10:35:49] (step=0046500) Train Loss: 6.3601, Train Steps/Sec: 0.89 + 1%|█▊ | 607/45917 [16:35<14:14:45, 1.13s/it][2025-04-24 10:36:17] (step=0046525) Train Loss: 6.3385, Train Steps/Sec: 0.90 + 1%|█▉ | 632/45917 [17:02<14:02:09, 1.12s/it][2025-04-24 10:36:45] (step=0046550) Train Loss: 6.3380, Train Steps/Sec: 0.90 + 1%|█▉ | 657/45917 [17:30<13:53:43, 1.11s/it][2025-04-24 10:37:12] (step=0046575) Train Loss: 6.2940, Train Steps/Sec: 0.90 + 1%|██ | 682/45917 [17:58<13:44:25, 1.09s/it][2025-04-24 10:37:40] (step=0046600) Train Loss: 6.3203, Train Steps/Sec: 0.90 + 2%|██ | 707/45917 [18:26<14:10:20, 1.13s/it][2025-04-24 10:38:08] (step=0046625) Train Loss: 6.3639, Train Steps/Sec: 0.90 + 2%|██▏ | 732/45917 [18:54<13:59:19, 1.11s/it][2025-04-24 10:38:36] (step=0046650) Train Loss: 6.3536, Train Steps/Sec: 0.89 + 2%|██▎ | 757/45917 [19:22<14:01:28, 1.12s/it][2025-04-24 10:39:04] (step=0046675) Train Loss: 6.3444, Train Steps/Sec: 0.90 + 2%|██▎ | 782/45917 [19:50<13:54:56, 1.11s/it][2025-04-24 10:39:32] (step=0046700) Train Loss: 6.3246, Train Steps/Sec: 0.89 + 2%|██▍ | 807/45917 [20:18<14:18:51, 1.14s/it][2025-04-24 10:40:00] (step=0046725) Train Loss: 6.3494, Train Steps/Sec: 0.89 + 2%|██▍ | 832/45917 [20:46<13:59:24, 1.12s/it][2025-04-24 10:40:28] (step=0046750) Train Loss: 6.3084, Train Steps/Sec: 0.89 + 2%|██▌ | 857/45917 [21:14<13:50:19, 1.11s/it][2025-04-24 10:40:56] (step=0046775) Train Loss: 6.3514, Train Steps/Sec: 0.90 + 2%|██▋ | 882/45917 [21:42<13:45:11, 1.10s/it][2025-04-24 10:41:24] (step=0046800) Train Loss: 6.3366, Train Steps/Sec: 0.89 + 2%|██▋ | 907/45917 [22:18<14:54:34, 1.19s/it][2025-04-24 10:42:00] (step=0046825) Train Loss: 6.3499, Train Steps/Sec: 0.69 + 2%|██▊ | 932/45917 [22:54<14:05:47, 1.13s/it][2025-04-24 10:42:37] (step=0046850) Train Loss: 6.3619, Train Steps/Sec: 0.69 + 2%|██▊ | 957/45917 [23:22<13:49:17, 1.11s/it][2025-04-24 10:43:04] (step=0046875) Train Loss: 6.3152, Train Steps/Sec: 0.90 + 2%|██▉ | 982/45917 [23:50<13:40:17, 1.10s/it][2025-04-24 10:43:32] (step=0046900) Train Loss: 6.3257, Train Steps/Sec: 0.89 + 2%|██▉ | 1007/45917 [24:18<14:02:28, 1.13s/it][2025-04-24 10:44:00] (step=0046925) Train Loss: 6.3094, Train Steps/Sec: 0.90 + 2%|███ | 1032/45917 [24:46<13:58:16, 1.12s/it][2025-04-24 10:44:28] (step=0046950) Train Loss: 6.3256, Train Steps/Sec: 0.90 + 2%|███▏ | 1057/45917 [25:14<13:55:59, 1.12s/it][2025-04-24 10:44:56] (step=0046975) Train Loss: 6.3151, Train Steps/Sec: 0.89 + 2%|███▏ | 1082/45917 [25:50<13:52:53, 1.11s/it][2025-04-24 10:45:33] (step=0047000) Train Loss: 6.3660, Train Steps/Sec: 0.69 + 2%|███▎ | 1107/45917 [26:18<14:11:31, 1.14s/it][2025-04-24 10:46:09] (step=0047025) Train Loss: 6.3163, Train Steps/Sec: 0.69 + 2%|███▎ | 1132/45917 [26:54<13:53:41, 1.12s/it][2025-04-24 10:46:36] (step=0047050) Train Loss: 6.3494, Train Steps/Sec: 0.90 + 3%|███▍ | 1157/45917 [27:22<13:41:15, 1.10s/it][2025-04-24 10:47:04] (step=0047075) Train Loss: 6.3276, Train Steps/Sec: 0.90 + 3%|███▌ | 1182/45917 [28:02<13:49:17, 1.11s/it][2025-04-24 10:47:44] (step=0047100) Train Loss: 6.3073, Train Steps/Sec: 0.63 + 3%|███▌ | 1207/45917 [28:30<14:12:23, 1.14s/it][2025-04-24 10:48:12] (step=0047125) Train Loss: 6.3430, Train Steps/Sec: 0.90 + 3%|███▋ | 1232/45917 [28:58<13:56:01, 1.12s/it][2025-04-24 10:48:40] (step=0047150) Train Loss: 6.4199, Train Steps/Sec: 0.89 + 3%|███▋ | 1257/45917 [29:26<13:50:53, 1.12s/it][2025-04-24 10:49:08] (step=0047175) Train Loss: 6.2874, Train Steps/Sec: 0.90 + 3%|███▊ | 1282/45917 [29:53<13:39:21, 1.10s/it][2025-04-24 10:49:36] (step=0047200) Train Loss: 6.3121, Train Steps/Sec: 0.89 + 3%|███▊ | 1307/45917 [30:21<14:02:15, 1.13s/it][2025-04-24 10:50:03] (step=0047225) Train Loss: 6.3384, Train Steps/Sec: 0.90 + 3%|███▉ | 1332/45917 [30:49<13:46:03, 1.11s/it][2025-04-24 10:50:31] (step=0047250) Train Loss: 6.3235, Train Steps/Sec: 0.90 + 3%|████ | 1357/45917 [31:17<13:44:40, 1.11s/it][2025-04-24 10:50:59] (step=0047275) Train Loss: 6.3145, Train Steps/Sec: 0.90 + 3%|████ | 1382/45917 [31:45<13:38:35, 1.10s/it][2025-04-24 10:51:27] (step=0047300) Train Loss: 6.3190, Train Steps/Sec: 0.90 + 3%|████▏ | 1407/45917 [32:13<13:57:17, 1.13s/it][2025-04-24 10:51:55] (step=0047325) Train Loss: 6.3214, Train Steps/Sec: 0.90 + 3%|████▏ | 1432/45917 [32:40<13:40:32, 1.11s/it][2025-04-24 10:52:23] (step=0047350) Train Loss: 6.2857, Train Steps/Sec: 0.90 + 3%|████▎ | 1457/45917 [33:08<13:34:28, 1.10s/it][2025-04-24 10:52:50] (step=0047375) Train Loss: 6.3091, Train Steps/Sec: 0.90 + 3%|████▍ | 1482/45917 [33:36<13:38:00, 1.10s/it][2025-04-24 10:53:18] (step=0047400) Train Loss: 6.3081, Train Steps/Sec: 0.89 + 3%|████▍ | 1507/45917 [34:04<14:00:31, 1.14s/it][2025-04-24 10:53:46] (step=0047425) Train Loss: 6.3627, Train Steps/Sec: 0.90 + 3%|████▌ | 1532/45917 [34:32<13:53:02, 1.13s/it][2025-04-24 10:54:14] (step=0047450) Train Loss: 6.3433, Train Steps/Sec: 0.89 + 3%|████▌ | 1557/45917 [35:00<13:44:54, 1.12s/it][2025-04-24 10:54:42] (step=0047475) Train Loss: 6.3532, Train Steps/Sec: 0.90 + 3%|████▋ | 1582/45917 [35:28<13:37:15, 1.11s/it][2025-04-24 10:55:10] (step=0047500) Train Loss: 6.3382, Train Steps/Sec: 0.89 + 3%|████▊ | 1607/45917 [35:56<13:58:46, 1.14s/it][2025-04-24 10:55:38] (step=0047525) Train Loss: 6.3671, Train Steps/Sec: 0.90 + 4%|████▊ | 1632/45917 [36:24<13:52:40, 1.13s/it][2025-04-24 10:56:06] (step=0047550) Train Loss: 6.3273, Train Steps/Sec: 0.90 + 4%|████▉ | 1657/45917 [36:52<13:35:56, 1.11s/it][2025-04-24 10:56:34] (step=0047575) Train Loss: 6.3097, Train Steps/Sec: 0.90 + 4%|████▉ | 1682/45917 [37:20<13:34:34, 1.10s/it][2025-04-24 10:57:02] (step=0047600) Train Loss: 6.3192, Train Steps/Sec: 0.89 + 4%|█████ | 1707/45917 [37:48<13:52:29, 1.13s/it][2025-04-24 10:57:30] (step=0047625) Train Loss: 6.3195, Train Steps/Sec: 0.90 + 4%|█████▏ | 1732/45917 [38:15<13:39:41, 1.11s/it][2025-04-24 10:57:58] (step=0047650) Train Loss: 6.3423, Train Steps/Sec: 0.90 + 4%|█████▏ | 1757/45917 [38:43<13:37:29, 1.11s/it][2025-04-24 10:58:25] (step=0047675) Train Loss: 6.3372, Train Steps/Sec: 0.90 + 4%|█████▎ | 1782/45917 [39:11<13:34:27, 1.11s/it][2025-04-24 10:58:53] (step=0047700) Train Loss: 6.3815, Train Steps/Sec: 0.89 + 4%|█████▎ | 1807/45917 [39:39<13:56:34, 1.14s/it][2025-04-24 10:59:21] (step=0047725) Train Loss: 6.3149, Train Steps/Sec: 0.89 + 4%|█████▍ | 1832/45917 [40:07<13:40:23, 1.12s/it][2025-04-24 10:59:49] (step=0047750) Train Loss: 6.3467, Train Steps/Sec: 0.90 + 4%|█████▌ | 1857/45917 [40:35<13:34:52, 1.11s/it][2025-04-24 11:00:17] (step=0047775) Train Loss: 6.2904, Train Steps/Sec: 0.90 + 4%|█████▌ | 1882/45917 [41:03<13:25:59, 1.10s/it][2025-04-24 11:00:45] (step=0047800) Train Loss: 6.3011, Train Steps/Sec: 0.89 + 4%|█████▋ | 1907/45917 [41:31<13:54:30, 1.14s/it][2025-04-24 11:01:13] (step=0047825) Train Loss: 6.2978, Train Steps/Sec: 0.90 + 4%|█████▋ | 1932/45917 [41:59<13:34:46, 1.11s/it][2025-04-24 11:01:41] (step=0047850) Train Loss: 6.3372, Train Steps/Sec: 0.90 + 4%|█████▊ | 1957/45917 [42:26<13:25:59, 1.10s/it][2025-04-24 11:02:09] (step=0047875) Train Loss: 6.3184, Train Steps/Sec: 0.90 + 4%|█████▊ | 1982/45917 [42:54<13:22:38, 1.10s/it][2025-04-24 11:02:36] (step=0047900) Train Loss: 6.3616, Train Steps/Sec: 0.90 + 4%|█████▉ | 2007/45917 [43:22<13:43:30, 1.13s/it][2025-04-24 11:03:04] (step=0047925) Train Loss: 6.3502, Train Steps/Sec: 0.90 + 4%|██████ | 2032/45917 [43:50<13:33:32, 1.11s/it][2025-04-24 11:03:32] (step=0047950) Train Loss: 6.2984, Train Steps/Sec: 0.90 + 4%|██████ | 2057/45917 [44:17<13:21:18, 1.10s/it][2025-04-24 11:04:00] (step=0047975) Train Loss: 6.3343, Train Steps/Sec: 0.90 + 5%|██████▏ | 2082/45917 [44:45<13:21:16, 1.10s/it][2025-04-24 11:04:28] (step=0048000) Train Loss: 6.3066, Train Steps/Sec: 0.90 +[2025-04-24 11:04:28] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 11:04:28] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 11:04:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.56s/it] +[2025-04-24 11:09:18] Finish Eval in 48000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.11s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 11:09:40] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0048000.pt +[2025-04-24 11:09:42] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0046000.pt + 5%|██████▏ | 2107/45917 [50:28<13:59:06, 1.15s/it][2025-04-24 11:10:10] (step=0048025) Train Loss: 6.3219, Train Steps/Sec: 0.07 + 5%|██████▎ | 2132/45917 [50:56<13:38:37, 1.12s/it][2025-04-24 11:10:38] (step=0048050) Train Loss: 6.2808, Train Steps/Sec: 0.90 + 5%|██████▍ | 2157/45917 [51:23<13:22:44, 1.10s/it][2025-04-24 11:11:06] (step=0048075) Train Loss: 6.3570, Train Steps/Sec: 0.90 + 5%|██████▍ | 2182/45917 [51:51<13:20:37, 1.10s/it][2025-04-24 11:11:34] (step=0048100) Train Loss: 6.3316, Train Steps/Sec: 0.89 + 5%|██████▌ | 2207/45917 [52:19<13:40:28, 1.13s/it][2025-04-24 11:12:01] (step=0048125) Train Loss: 6.3250, Train Steps/Sec: 0.90 + 5%|██████▌ | 2232/45917 [52:47<13:40:02, 1.13s/it][2025-04-24 11:12:29] (step=0048150) Train Loss: 6.3035, Train Steps/Sec: 0.90 + 5%|██████▋ | 2257/45917 [53:15<13:25:32, 1.11s/it][2025-04-24 11:12:57] (step=0048175) Train Loss: 6.3382, Train Steps/Sec: 0.90 + 5%|██████▊ | 2282/45917 [53:42<13:10:45, 1.09s/it][2025-04-24 11:13:25] (step=0048200) Train Loss: 6.3257, Train Steps/Sec: 0.90 + 5%|██████▊ | 2307/45917 [54:11<13:46:17, 1.14s/it][2025-04-24 11:13:53] (step=0048225) Train Loss: 6.3291, Train Steps/Sec: 0.90 + 5%|██████▉ | 2332/45917 [54:38<13:28:31, 1.11s/it][2025-04-24 11:14:20] (step=0048250) Train Loss: 6.3508, Train Steps/Sec: 0.90 + 5%|██████▉ | 2357/45917 [55:06<13:11:31, 1.09s/it][2025-04-24 11:14:48] (step=0048275) Train Loss: 6.3444, Train Steps/Sec: 0.90 + 5%|███████ | 2382/45917 [55:34<13:17:56, 1.10s/it][2025-04-24 11:15:16] (step=0048300) Train Loss: 6.3262, Train Steps/Sec: 0.90 + 5%|███████▏ | 2407/45917 [56:01<13:42:23, 1.13s/it][2025-04-24 11:15:44] (step=0048325) Train Loss: 6.3984, Train Steps/Sec: 0.90 + 5%|███████▏ | 2432/45917 [56:29<13:28:20, 1.12s/it][2025-04-24 11:16:11] (step=0048350) Train Loss: 6.3229, Train Steps/Sec: 0.90 + 5%|███████▎ | 2457/45917 [56:57<13:19:15, 1.10s/it][2025-04-24 11:16:39] (step=0048375) Train Loss: 6.3290, Train Steps/Sec: 0.90 + 5%|███████▎ | 2482/45917 [57:25<13:09:54, 1.09s/it][2025-04-24 11:17:07] (step=0048400) Train Loss: 6.3452, Train Steps/Sec: 0.90 + 5%|███████▍ | 2507/45917 [57:53<13:38:49, 1.13s/it][2025-04-24 11:17:35] (step=0048425) Train Loss: 6.3082, Train Steps/Sec: 0.90 + 6%|███████▍ | 2532/45917 [58:20<13:26:14, 1.12s/it][2025-04-24 11:18:03] (step=0048450) Train Loss: 6.3349, Train Steps/Sec: 0.90 + 6%|███████▌ | 2557/45917 [58:48<13:17:09, 1.10s/it][2025-04-24 11:18:30] (step=0048475) Train Loss: 6.3288, Train Steps/Sec: 0.90 + 6%|███████▋ | 2582/45917 [59:16<13:17:08, 1.10s/it][2025-04-24 11:18:58] (step=0048500) Train Loss: 6.3716, Train Steps/Sec: 0.90 + 6%|███████▋ | 2607/45917 [59:44<13:30:45, 1.12s/it][2025-04-24 11:19:26] (step=0048525) Train Loss: 6.3712, Train Steps/Sec: 0.90 + 6%|███████▋ | 2632/45917 [1:00:12<13:17:58, 1.11s/it][2025-04-24 11:19:54] (step=0048550) Train Loss: 6.3208, Train Steps/Sec: 0.90 + 6%|███████▊ | 2657/45917 [1:00:39<13:12:01, 1.10s/it][2025-04-24 11:20:21] (step=0048575) Train Loss: 6.3165, Train Steps/Sec: 0.90 + 6%|███████▊ | 2682/45917 [1:01:07<13:22:25, 1.11s/it][2025-04-24 11:20:50] (step=0048600) Train Loss: 6.3174, Train Steps/Sec: 0.89 + 6%|███████▉ | 2707/45917 [1:01:35<13:36:54, 1.13s/it][2025-04-24 11:21:17] (step=0048625) Train Loss: 6.3253, Train Steps/Sec: 0.90 + 6%|███████▉ | 2732/45917 [1:02:03<13:22:38, 1.12s/it][2025-04-24 11:21:45] (step=0048650) Train Loss: 6.3361, Train Steps/Sec: 0.90 + 6%|████████ | 2757/45917 [1:02:31<13:11:49, 1.10s/it][2025-04-24 11:22:13] (step=0048675) Train Loss: 6.3495, Train Steps/Sec: 0.90 + 6%|████████ | 2782/45917 [1:02:58<13:13:39, 1.10s/it][2025-04-24 11:22:41] (step=0048700) Train Loss: 6.3831, Train Steps/Sec: 0.90 + 6%|████████▏ | 2807/45917 [1:03:26<13:28:20, 1.13s/it][2025-04-24 11:23:09] (step=0048725) Train Loss: 6.3515, Train Steps/Sec: 0.90 + 6%|████████▎ | 2832/45917 [1:03:54<13:16:32, 1.11s/it][2025-04-24 11:23:36] (step=0048750) Train Loss: 6.3181, Train Steps/Sec: 0.90 + 6%|████████▎ | 2857/45917 [1:04:22<13:13:29, 1.11s/it][2025-04-24 11:24:04] (step=0048775) Train Loss: 6.3047, Train Steps/Sec: 0.90 + 6%|████████▍ | 2882/45917 [1:04:50<13:09:04, 1.10s/it][2025-04-24 11:24:32] (step=0048800) Train Loss: 6.3748, Train Steps/Sec: 0.90 + 6%|████████▍ | 2907/45917 [1:05:18<13:30:50, 1.13s/it][2025-04-24 11:25:00] (step=0048825) Train Loss: 6.3462, Train Steps/Sec: 0.90 + 6%|████████▌ | 2932/45917 [1:05:45<13:14:49, 1.11s/it][2025-04-24 11:25:27] (step=0048850) Train Loss: 6.3012, Train Steps/Sec: 0.90 + 6%|████████▋ | 2957/45917 [1:06:13<13:07:26, 1.10s/it][2025-04-24 11:25:55] (step=0048875) Train Loss: 6.3173, Train Steps/Sec: 0.90 + 6%|████████▋ | 2982/45917 [1:06:41<13:04:53, 1.10s/it][2025-04-24 11:26:23] (step=0048900) Train Loss: 6.3041, Train Steps/Sec: 0.90 + 7%|████████▊ | 3007/45917 [1:07:09<13:24:04, 1.12s/it][2025-04-24 11:26:51] (step=0048925) Train Loss: 6.3295, Train Steps/Sec: 0.90 + 7%|████████▊ | 3032/45917 [1:07:36<13:09:19, 1.10s/it][2025-04-24 11:27:18] (step=0048950) Train Loss: 6.3037, Train Steps/Sec: 0.90 + 7%|████████▉ | 3057/45917 [1:08:04<13:03:20, 1.10s/it][2025-04-24 11:27:46] (step=0048975) Train Loss: 6.3249, Train Steps/Sec: 0.91 + 7%|████████▉ | 3082/45917 [1:08:32<13:10:55, 1.11s/it][2025-04-24 11:28:14] (step=0049000) Train Loss: 6.3676, Train Steps/Sec: 0.89 + 7%|█████████ | 3107/45917 [1:09:00<13:21:23, 1.12s/it][2025-04-24 11:28:42] (step=0049025) Train Loss: 6.3482, Train Steps/Sec: 0.90 + 7%|█████████▏ | 3132/45917 [1:09:27<13:12:39, 1.11s/it][2025-04-24 11:29:09] (step=0049050) Train Loss: 6.3343, Train Steps/Sec: 0.90 + 7%|█████████▏ | 3157/45917 [1:09:55<13:08:22, 1.11s/it][2025-04-24 11:29:37] (step=0049075) Train Loss: 6.3205, Train Steps/Sec: 0.90 + 7%|█████████▎ | 3182/45917 [1:10:23<13:02:01, 1.10s/it][2025-04-24 11:30:05] (step=0049100) Train Loss: 6.2988, Train Steps/Sec: 0.90 + 7%|█████████▎ | 3207/45917 [1:10:51<13:17:24, 1.12s/it][2025-04-24 11:30:33] (step=0049125) Train Loss: 6.3326, Train Steps/Sec: 0.90 + 7%|█████████▍ | 3232/45917 [1:11:18<13:13:28, 1.12s/it][2025-04-24 11:31:01] (step=0049150) Train Loss: 6.3608, Train Steps/Sec: 0.90 + 7%|█████████▌ | 3257/45917 [1:11:46<13:02:31, 1.10s/it][2025-04-24 11:31:28] (step=0049175) Train Loss: 6.2732, Train Steps/Sec: 0.90 + 7%|█████████▌ | 3282/45917 [1:12:14<12:55:51, 1.09s/it][2025-04-24 11:31:56] (step=0049200) Train Loss: 6.3867, Train Steps/Sec: 0.90 + 7%|█████████▋ | 3307/45917 [1:12:42<13:25:49, 1.13s/it][2025-04-24 11:32:24] (step=0049225) Train Loss: 6.3234, Train Steps/Sec: 0.90 + 7%|█████████▋ | 3332/45917 [1:13:10<13:10:51, 1.11s/it][2025-04-24 11:32:52] (step=0049250) Train Loss: 6.3332, Train Steps/Sec: 0.90 + 7%|█████████▊ | 3357/45917 [1:13:47<14:27:04, 1.22s/it][2025-04-24 11:33:29] (step=0049275) Train Loss: 6.3194, Train Steps/Sec: 0.68 + 7%|█████████▊ | 3382/45917 [1:14:14<13:01:08, 1.10s/it][2025-04-24 11:33:57] (step=0049300) Train Loss: 6.3244, Train Steps/Sec: 0.90 + 7%|█████████▉ | 3407/45917 [1:14:51<17:03:30, 1.44s/it][2025-04-24 11:34:33] (step=0049325) Train Loss: 6.3273, Train Steps/Sec: 0.68 + 7%|██████████ | 3432/45917 [1:15:19<13:01:34, 1.10s/it][2025-04-24 11:35:01] (step=0049350) Train Loss: 6.2834, Train Steps/Sec: 0.90 + 8%|██████████ | 3457/45917 [1:15:47<13:04:20, 1.11s/it][2025-04-24 11:35:29] (step=0049375) Train Loss: 6.2591, Train Steps/Sec: 0.90 + 8%|██████████▏ | 3482/45917 [1:16:14<12:59:01, 1.10s/it][2025-04-24 11:35:57] (step=0049400) Train Loss: 6.3498, Train Steps/Sec: 0.90 + 8%|██████████▏ | 3507/45917 [1:16:42<13:28:16, 1.14s/it][2025-04-24 11:36:24] (step=0049425) Train Loss: 6.3043, Train Steps/Sec: 0.90 + 8%|██████████▎ | 3532/45917 [1:17:10<13:01:59, 1.11s/it][2025-04-24 11:36:52] (step=0049450) Train Loss: 6.3492, Train Steps/Sec: 0.90 + 8%|██████████▍ | 3557/45917 [1:17:38<13:03:12, 1.11s/it][2025-04-24 11:37:20] (step=0049475) Train Loss: 6.3791, Train Steps/Sec: 0.90 + 8%|██████████▍ | 3582/45917 [1:18:05<12:56:08, 1.10s/it][2025-04-24 11:37:48] (step=0049500) Train Loss: 6.3105, Train Steps/Sec: 0.90 + 8%|██████████▌ | 3607/45917 [1:18:33<13:16:15, 1.13s/it][2025-04-24 11:38:15] (step=0049525) Train Loss: 6.3236, Train Steps/Sec: 0.90 + 8%|██████████▌ | 3632/45917 [1:19:01<13:10:13, 1.12s/it][2025-04-24 11:38:43] (step=0049550) Train Loss: 6.2718, Train Steps/Sec: 0.90 + 8%|██████████▋ | 3657/45917 [1:19:29<12:54:11, 1.10s/it][2025-04-24 11:39:11] (step=0049575) Train Loss: 6.3338, Train Steps/Sec: 0.90 + 8%|██████████▋ | 3682/45917 [1:19:57<12:51:07, 1.10s/it][2025-04-24 11:39:39] (step=0049600) Train Loss: 6.3131, Train Steps/Sec: 0.90 + 8%|██████████▊ | 3707/45917 [1:20:25<13:13:04, 1.13s/it][2025-04-24 11:40:07] (step=0049625) Train Loss: 6.3264, Train Steps/Sec: 0.90 + 8%|██████████▉ | 3732/45917 [1:20:52<13:01:06, 1.11s/it][2025-04-24 11:40:34] (step=0049650) Train Loss: 6.2999, Train Steps/Sec: 0.90 + 8%|██████████▉ | 3757/45917 [1:21:20<13:00:29, 1.11s/it][2025-04-24 11:41:02] (step=0049675) Train Loss: 6.3042, Train Steps/Sec: 0.90 + 8%|███████████ | 3782/45917 [1:21:48<12:53:22, 1.10s/it][2025-04-24 11:41:30] (step=0049700) Train Loss: 6.2997, Train Steps/Sec: 0.90 + 8%|███████████ | 3807/45917 [1:22:16<13:12:31, 1.13s/it][2025-04-24 11:41:58] (step=0049725) Train Loss: 6.3022, Train Steps/Sec: 0.90 + 8%|███████████▏ | 3832/45917 [1:22:43<12:58:16, 1.11s/it][2025-04-24 11:42:25] (step=0049750) Train Loss: 6.2805, Train Steps/Sec: 0.90 + 8%|███████████▎ | 3857/45917 [1:23:11<12:56:23, 1.11s/it][2025-04-24 11:42:53] (step=0049775) Train Loss: 6.3330, Train Steps/Sec: 0.90 + 8%|███████████▎ | 3882/45917 [1:23:39<12:54:01, 1.10s/it][2025-04-24 11:43:21] (step=0049800) Train Loss: 6.2872, Train Steps/Sec: 0.90 + 9%|███████████▍ | 3907/45917 [1:24:07<13:07:38, 1.12s/it][2025-04-24 11:43:49] (step=0049825) Train Loss: 6.3754, Train Steps/Sec: 0.90 + 9%|███████████▍ | 3932/45917 [1:24:34<12:52:37, 1.10s/it][2025-04-24 11:44:17] (step=0049850) Train Loss: 6.3196, Train Steps/Sec: 0.90 + 9%|███████████▌ | 3957/45917 [1:25:02<12:55:49, 1.11s/it][2025-04-24 11:44:44] (step=0049875) Train Loss: 6.3179, Train Steps/Sec: 0.90 + 9%|███████████▌ | 3982/45917 [1:25:30<12:43:28, 1.09s/it][2025-04-24 11:45:12] (step=0049900) Train Loss: 6.4011, Train Steps/Sec: 0.90 + 9%|███████████▋ | 4007/45917 [1:25:58<13:07:23, 1.13s/it][2025-04-24 11:45:40] (step=0049925) Train Loss: 6.3136, Train Steps/Sec: 0.90 + 9%|███████████▊ | 4032/45917 [1:26:26<13:03:39, 1.12s/it][2025-04-24 11:46:08] (step=0049950) Train Loss: 6.3148, Train Steps/Sec: 0.90 + 9%|███████████▊ | 4057/45917 [1:26:54<12:47:46, 1.10s/it][2025-04-24 11:46:36] (step=0049975) Train Loss: 6.3124, Train Steps/Sec: 0.90 + 9%|███████████▉ | 4082/45917 [1:27:21<12:43:52, 1.10s/it][2025-04-24 11:47:04] (step=0050000) Train Loss: 6.3724, Train Steps/Sec: 0.90 +[2025-04-24 11:47:04] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 11:47:04] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 11:47:04] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 59.18s/it] +[2025-04-24 11:51:51] Finish Eval in 50000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.74s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 11:52:13] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0050000.pt +[2025-04-24 11:52:15] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0048000.pt + 9%|███████████▉ | 4107/45917 [1:33:01<13:28:31, 1.16s/it][2025-04-24 11:52:43] (step=0050025) Train Loss: 6.3212, Train Steps/Sec: 0.07 + 9%|████████████ | 4132/45917 [1:33:29<12:59:51, 1.12s/it][2025-04-24 11:53:11] (step=0050050) Train Loss: 6.3186, Train Steps/Sec: 0.90 + 9%|████████████▏ | 4157/45917 [1:33:56<12:48:16, 1.10s/it][2025-04-24 11:53:38] (step=0050075) Train Loss: 6.2713, Train Steps/Sec: 0.90 + 9%|████████████▏ | 4182/45917 [1:34:24<12:44:24, 1.10s/it][2025-04-24 11:54:06] (step=0050100) Train Loss: 6.3054, Train Steps/Sec: 0.89 + 9%|████████████▎ | 4207/45917 [1:34:52<13:08:06, 1.13s/it][2025-04-24 11:54:34] (step=0050125) Train Loss: 6.3902, Train Steps/Sec: 0.90 + 9%|████████████▎ | 4232/45917 [1:35:20<12:50:50, 1.11s/it][2025-04-24 11:55:02] (step=0050150) Train Loss: 6.3399, Train Steps/Sec: 0.90 + 9%|████████████▍ | 4257/45917 [1:35:48<12:47:26, 1.11s/it][2025-04-24 11:55:30] (step=0050175) Train Loss: 6.3470, Train Steps/Sec: 0.90 + 9%|████████████▍ | 4282/45917 [1:36:15<12:45:27, 1.10s/it][2025-04-24 11:55:58] (step=0050200) Train Loss: 6.2940, Train Steps/Sec: 0.90 + 9%|████████████▌ | 4307/45917 [1:36:43<12:58:29, 1.12s/it][2025-04-24 11:56:25] (step=0050225) Train Loss: 6.3134, Train Steps/Sec: 0.90 + 9%|████████████▋ | 4332/45917 [1:37:11<12:52:31, 1.11s/it][2025-04-24 11:56:53] (step=0050250) Train Loss: 6.3004, Train Steps/Sec: 0.90 + 9%|████████████▋ | 4357/45917 [1:37:39<12:46:10, 1.11s/it][2025-04-24 11:57:21] (step=0050275) Train Loss: 6.3277, Train Steps/Sec: 0.90 + 10%|████████████▊ | 4382/45917 [1:38:07<12:38:04, 1.10s/it][2025-04-24 11:57:49] (step=0050300) Train Loss: 6.3506, Train Steps/Sec: 0.90 + 10%|████████████▊ | 4407/45917 [1:38:34<12:57:03, 1.12s/it][2025-04-24 11:58:16] (step=0050325) Train Loss: 6.3219, Train Steps/Sec: 0.91 + 10%|████████████▉ | 4432/45917 [1:39:02<12:49:31, 1.11s/it][2025-04-24 11:58:44] (step=0050350) Train Loss: 6.3423, Train Steps/Sec: 0.90 + 10%|█████████████ | 4457/45917 [1:39:30<12:51:54, 1.12s/it][2025-04-24 11:59:12] (step=0050375) Train Loss: 6.3489, Train Steps/Sec: 0.90 + 10%|█████████████ | 4482/45917 [1:39:58<12:44:29, 1.11s/it][2025-04-24 11:59:40] (step=0050400) Train Loss: 6.3229, Train Steps/Sec: 0.90 + 10%|█████████████▏ | 4507/45917 [1:40:25<13:00:13, 1.13s/it][2025-04-24 12:00:08] (step=0050425) Train Loss: 6.2549, Train Steps/Sec: 0.90 + 10%|█████████████▏ | 4532/45917 [1:40:53<12:48:50, 1.11s/it][2025-04-24 12:00:35] (step=0050450) Train Loss: 6.2859, Train Steps/Sec: 0.90 + 10%|█████████████▎ | 4557/45917 [1:41:21<12:39:17, 1.10s/it][2025-04-24 12:01:03] (step=0050475) Train Loss: 6.3517, Train Steps/Sec: 0.89 + 10%|█████████████▎ | 4582/45917 [1:41:49<12:31:39, 1.09s/it][2025-04-24 12:01:31] (step=0050500) Train Loss: 6.3265, Train Steps/Sec: 0.90 + 10%|█████████████▍ | 4607/45917 [1:42:17<12:50:01, 1.12s/it][2025-04-24 12:01:59] (step=0050525) Train Loss: 6.3436, Train Steps/Sec: 0.91 + 10%|█████████████▌ | 4632/45917 [1:42:44<12:44:04, 1.11s/it][2025-04-24 12:02:27] (step=0050550) Train Loss: 6.3241, Train Steps/Sec: 0.90 + 10%|█████████████▌ | 4657/45917 [1:43:12<12:42:21, 1.11s/it][2025-04-24 12:02:54] (step=0050575) Train Loss: 6.2906, Train Steps/Sec: 0.90 + 10%|█████████████▋ | 4682/45917 [1:43:40<12:34:27, 1.10s/it][2025-04-24 12:03:22] (step=0050600) Train Loss: 6.2611, Train Steps/Sec: 0.90 + 10%|█████████████▋ | 4707/45917 [1:44:08<12:56:16, 1.13s/it][2025-04-24 12:03:50] (step=0050625) Train Loss: 6.3184, Train Steps/Sec: 0.90 + 10%|█████████████▊ | 4732/45917 [1:44:35<12:43:14, 1.11s/it][2025-04-24 12:04:18] (step=0050650) Train Loss: 6.2943, Train Steps/Sec: 0.90 + 10%|█████████████▉ | 4757/45917 [1:45:03<12:33:46, 1.10s/it][2025-04-24 12:04:45] (step=0050675) Train Loss: 6.3501, Train Steps/Sec: 0.90 + 10%|█████████████▉ | 4782/45917 [1:45:31<12:29:51, 1.09s/it][2025-04-24 12:05:13] (step=0050700) Train Loss: 6.3250, Train Steps/Sec: 0.90 + 10%|██████████████ | 4807/45917 [1:45:59<12:49:45, 1.12s/it][2025-04-24 12:05:41] (step=0050725) Train Loss: 6.3057, Train Steps/Sec: 0.90 + 11%|██████████████ | 4832/45917 [1:46:27<12:41:09, 1.11s/it][2025-04-24 12:06:09] (step=0050750) Train Loss: 6.2920, Train Steps/Sec: 0.90 + 11%|██████████████▏ | 4857/45917 [1:46:54<12:36:20, 1.11s/it][2025-04-24 12:06:36] (step=0050775) Train Loss: 6.3145, Train Steps/Sec: 0.90 + 11%|██████████████▏ | 4882/45917 [1:47:22<12:44:57, 1.12s/it][2025-04-24 12:07:04] (step=0050800) Train Loss: 6.3251, Train Steps/Sec: 0.89 + 11%|██████████████▎ | 4907/45917 [1:47:50<12:56:16, 1.14s/it][2025-04-24 12:07:32] (step=0050825) Train Loss: 6.3223, Train Steps/Sec: 0.90 + 11%|██████████████▍ | 4932/45917 [1:48:18<12:38:04, 1.11s/it][2025-04-24 12:08:00] (step=0050850) Train Loss: 6.3537, Train Steps/Sec: 0.90 + 11%|██████████████▍ | 4957/45917 [1:48:46<12:31:49, 1.10s/it][2025-04-24 12:08:28] (step=0050875) Train Loss: 6.3047, Train Steps/Sec: 0.90 + 11%|██████████████▌ | 4982/45917 [1:49:13<12:25:00, 1.09s/it][2025-04-24 12:08:56] (step=0050900) Train Loss: 6.3068, Train Steps/Sec: 0.90 + 11%|██████████████▌ | 5007/45917 [1:49:41<12:49:09, 1.13s/it][2025-04-24 12:09:23] (step=0050925) Train Loss: 6.3263, Train Steps/Sec: 0.90 + 11%|██████████████▋ | 5032/45917 [1:50:09<12:36:54, 1.11s/it][2025-04-24 12:09:51] (step=0050950) Train Loss: 6.3385, Train Steps/Sec: 0.90 + 11%|██████████████▊ | 5057/45917 [1:50:37<12:35:34, 1.11s/it][2025-04-24 12:10:19] (step=0050975) Train Loss: 6.3300, Train Steps/Sec: 0.90 + 11%|██████████████▊ | 5082/45917 [1:51:05<12:30:54, 1.10s/it][2025-04-24 12:10:47] (step=0051000) Train Loss: 6.3108, Train Steps/Sec: 0.90 + 11%|██████████████▉ | 5107/45917 [1:51:32<12:48:00, 1.13s/it][2025-04-24 12:11:15] (step=0051025) Train Loss: 6.3138, Train Steps/Sec: 0.90 + 11%|██████████████▉ | 5132/45917 [1:52:00<12:34:37, 1.11s/it][2025-04-24 12:11:42] (step=0051050) Train Loss: 6.3403, Train Steps/Sec: 0.90 + 11%|███████████████ | 5157/45917 [1:52:28<12:33:53, 1.11s/it][2025-04-24 12:12:10] (step=0051075) Train Loss: 6.3152, Train Steps/Sec: 0.90 + 11%|███████████████ | 5182/45917 [1:52:55<12:23:44, 1.10s/it][2025-04-24 12:12:38] (step=0051100) Train Loss: 6.3333, Train Steps/Sec: 0.90 + 11%|███████████████▏ | 5207/45917 [1:53:23<12:46:27, 1.13s/it][2025-04-24 12:13:06] (step=0051125) Train Loss: 6.3886, Train Steps/Sec: 0.90 + 11%|███████████████▎ | 5232/45917 [1:53:51<12:34:27, 1.11s/it][2025-04-24 12:13:33] (step=0051150) Train Loss: 6.3287, Train Steps/Sec: 0.90 + 11%|███████████████▎ | 5257/45917 [1:54:19<12:26:40, 1.10s/it][2025-04-24 12:14:01] (step=0051175) Train Loss: 6.3211, Train Steps/Sec: 0.90 + 12%|███████████████▍ | 5282/45917 [1:54:47<12:31:12, 1.11s/it][2025-04-24 12:14:29] (step=0051200) Train Loss: 6.3093, Train Steps/Sec: 0.90 + 12%|███████████████▍ | 5307/45917 [1:55:14<12:43:49, 1.13s/it][2025-04-24 12:14:57] (step=0051225) Train Loss: 6.2947, Train Steps/Sec: 0.90 + 12%|███████████████▌ | 5332/45917 [1:55:42<12:34:51, 1.12s/it][2025-04-24 12:15:24] (step=0051250) Train Loss: 6.2941, Train Steps/Sec: 0.90 + 12%|███████████████▋ | 5357/45917 [1:56:10<12:27:59, 1.11s/it][2025-04-24 12:15:52] (step=0051275) Train Loss: 6.3319, Train Steps/Sec: 0.90 + 12%|███████████████▋ | 5382/45917 [1:56:38<12:20:04, 1.10s/it][2025-04-24 12:16:20] (step=0051300) Train Loss: 6.3239, Train Steps/Sec: 0.90 + 12%|███████████████▊ | 5407/45917 [1:57:06<12:46:56, 1.14s/it][2025-04-24 12:16:48] (step=0051325) Train Loss: 6.3408, Train Steps/Sec: 0.90 + 12%|███████████████▊ | 5432/45917 [1:57:33<12:31:47, 1.11s/it][2025-04-24 12:17:15] (step=0051350) Train Loss: 6.2678, Train Steps/Sec: 0.90 + 12%|███████████████▉ | 5457/45917 [1:58:01<12:24:41, 1.10s/it][2025-04-24 12:17:43] (step=0051375) Train Loss: 6.3252, Train Steps/Sec: 0.90 + 12%|███████████████▉ | 5482/45917 [1:58:29<12:18:00, 1.10s/it][2025-04-24 12:18:11] (step=0051400) Train Loss: 6.2915, Train Steps/Sec: 0.90 + 12%|████████████████ | 5507/45917 [1:58:57<12:43:09, 1.13s/it][2025-04-24 12:18:39] (step=0051425) Train Loss: 6.3206, Train Steps/Sec: 0.90 + 12%|████████████████▏ | 5532/45917 [1:59:24<12:25:18, 1.11s/it][2025-04-24 12:19:06] (step=0051450) Train Loss: 6.3261, Train Steps/Sec: 0.90 + 12%|████████████████▏ | 5557/45917 [1:59:52<12:25:59, 1.11s/it][2025-04-24 12:19:34] (step=0051475) Train Loss: 6.2738, Train Steps/Sec: 0.90 + 12%|████████████████▎ | 5582/45917 [2:00:20<12:20:38, 1.10s/it][2025-04-24 12:20:02] (step=0051500) Train Loss: 6.2833, Train Steps/Sec: 0.90 + 12%|████████████████▎ | 5607/45917 [2:00:48<12:38:14, 1.13s/it][2025-04-24 12:20:30] (step=0051525) Train Loss: 6.3347, Train Steps/Sec: 0.90 + 12%|████████████████▍ | 5632/45917 [2:01:15<12:31:01, 1.12s/it][2025-04-24 12:20:58] (step=0051550) Train Loss: 6.3386, Train Steps/Sec: 0.90 + 12%|████████████████▌ | 5657/45917 [2:01:43<12:17:32, 1.10s/it][2025-04-24 12:21:25] (step=0051575) Train Loss: 6.3435, Train Steps/Sec: 0.90 + 12%|████████████████▌ | 5682/45917 [2:02:11<12:16:59, 1.10s/it][2025-04-24 12:21:53] (step=0051600) Train Loss: 6.3014, Train Steps/Sec: 0.90 + 12%|████████████████▋ | 5707/45917 [2:02:39<12:38:26, 1.13s/it][2025-04-24 12:22:21] (step=0051625) Train Loss: 6.3429, Train Steps/Sec: 0.90 + 12%|████████████████▋ | 5732/45917 [2:03:06<12:29:31, 1.12s/it][2025-04-24 12:22:48] (step=0051650) Train Loss: 6.2649, Train Steps/Sec: 0.90 + 13%|████████████████▊ | 5757/45917 [2:03:34<12:20:43, 1.11s/it][2025-04-24 12:23:16] (step=0051675) Train Loss: 6.2951, Train Steps/Sec: 0.90 + 13%|████████████████▊ | 5782/45917 [2:04:02<12:10:46, 1.09s/it][2025-04-24 12:23:44] (step=0051700) Train Loss: 6.2973, Train Steps/Sec: 0.90 + 13%|████████████████▉ | 5807/45917 [2:04:30<12:34:07, 1.13s/it][2025-04-24 12:24:12] (step=0051725) Train Loss: 6.3230, Train Steps/Sec: 0.90 + 13%|█████████████████ | 5832/45917 [2:04:57<12:20:01, 1.11s/it][2025-04-24 12:24:39] (step=0051750) Train Loss: 6.3154, Train Steps/Sec: 0.90 + 13%|█████████████████ | 5857/45917 [2:05:25<12:18:57, 1.11s/it][2025-04-24 12:25:07] (step=0051775) Train Loss: 6.3302, Train Steps/Sec: 0.90 + 13%|█████████████████▏ | 5882/45917 [2:05:53<12:09:22, 1.09s/it][2025-04-24 12:25:35] (step=0051800) Train Loss: 6.3105, Train Steps/Sec: 0.90 + 13%|█████████████████▏ | 5907/45917 [2:06:21<12:29:50, 1.12s/it][2025-04-24 12:26:03] (step=0051825) Train Loss: 6.3258, Train Steps/Sec: 0.90 + 13%|█████████████████▎ | 5932/45917 [2:06:48<12:21:00, 1.11s/it][2025-04-24 12:26:30] (step=0051850) Train Loss: 6.3720, Train Steps/Sec: 0.90 + 13%|█████████████████▍ | 5957/45917 [2:07:16<12:18:58, 1.11s/it][2025-04-24 12:26:58] (step=0051875) Train Loss: 6.3295, Train Steps/Sec: 0.90 + 13%|█████████████████▍ | 5982/45917 [2:07:44<12:09:13, 1.10s/it][2025-04-24 12:27:26] (step=0051900) Train Loss: 6.3398, Train Steps/Sec: 0.90 + 13%|█████████████████▌ | 6007/45917 [2:08:12<12:26:02, 1.12s/it][2025-04-24 12:27:54] (step=0051925) Train Loss: 6.3120, Train Steps/Sec: 0.90 + 13%|█████████████████▌ | 6032/45917 [2:08:39<12:18:45, 1.11s/it][2025-04-24 12:28:21] (step=0051950) Train Loss: 6.2774, Train Steps/Sec: 0.90 + 13%|█████████████████▋ | 6057/45917 [2:09:07<12:23:59, 1.12s/it][2025-04-24 12:28:49] (step=0051975) Train Loss: 6.3016, Train Steps/Sec: 0.90 + 13%|█████████████████▋ | 6082/45917 [2:09:35<12:14:17, 1.11s/it][2025-04-24 12:29:17] (step=0052000) Train Loss: 6.3534, Train Steps/Sec: 0.89 +[2025-04-24 12:29:17] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 12:29:17] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 12:29:17] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.52s/it] +[2025-04-24 12:34:06] Finish Eval in 52000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 58.96s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 12:34:28] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0052000.pt +[2025-04-24 12:34:30] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0050000.pt + 13%|█████████████████▊ | 6107/45917 [2:15:16<12:42:24, 1.15s/it][2025-04-24 12:34:58] (step=0052025) Train Loss: 6.3592, Train Steps/Sec: 0.07 + 13%|█████████████████▉ | 6132/45917 [2:15:43<12:18:16, 1.11s/it][2025-04-24 12:35:26] (step=0052050) Train Loss: 6.3231, Train Steps/Sec: 0.90 + 13%|█████████████████▉ | 6157/45917 [2:16:11<12:14:35, 1.11s/it][2025-04-24 12:35:53] (step=0052075) Train Loss: 6.3368, Train Steps/Sec: 0.90 + 13%|██████████████████ | 6182/45917 [2:16:39<12:01:12, 1.09s/it][2025-04-24 12:36:21] (step=0052100) Train Loss: 6.3815, Train Steps/Sec: 0.90 + 14%|██████████████████ | 6207/45917 [2:17:06<12:24:57, 1.13s/it][2025-04-24 12:36:49] (step=0052125) Train Loss: 6.3037, Train Steps/Sec: 0.90 + 14%|██████████████████▏ | 6232/45917 [2:17:34<12:11:10, 1.11s/it][2025-04-24 12:37:16] (step=0052150) Train Loss: 6.2954, Train Steps/Sec: 0.91 + 14%|██████████████████▎ | 6257/45917 [2:18:02<12:10:02, 1.10s/it][2025-04-24 12:37:44] (step=0052175) Train Loss: 6.3231, Train Steps/Sec: 0.90 + 14%|██████████████████▎ | 6282/45917 [2:18:30<12:10:32, 1.11s/it][2025-04-24 12:38:12] (step=0052200) Train Loss: 6.3416, Train Steps/Sec: 0.89 + 14%|██████████████████▍ | 6307/45917 [2:18:57<12:27:39, 1.13s/it][2025-04-24 12:38:40] (step=0052225) Train Loss: 6.3468, Train Steps/Sec: 0.90 + 14%|██████████████████▍ | 6332/45917 [2:19:25<12:24:36, 1.13s/it][2025-04-24 12:39:08] (step=0052250) Train Loss: 6.3154, Train Steps/Sec: 0.90 + 14%|██████████████████▌ | 6357/45917 [2:19:59<12:26:29, 1.13s/it][2025-04-24 12:39:42] (step=0052275) Train Loss: 6.3101, Train Steps/Sec: 0.74 + 14%|██████████████████▌ | 6382/45917 [2:20:34<14:31:10, 1.32s/it][2025-04-24 12:40:16] (step=0052300) Train Loss: 6.3449, Train Steps/Sec: 0.73 + 14%|██████████████████▋ | 6407/45917 [2:21:02<12:40:05, 1.15s/it][2025-04-24 12:40:44] (step=0052325) Train Loss: 6.2886, Train Steps/Sec: 0.89 + 14%|██████████████████▊ | 6432/45917 [2:21:30<12:10:34, 1.11s/it][2025-04-24 12:41:12] (step=0052350) Train Loss: 6.3419, Train Steps/Sec: 0.90 + 14%|██████████████████▊ | 6457/45917 [2:21:57<12:07:31, 1.11s/it][2025-04-24 12:41:40] (step=0052375) Train Loss: 6.3503, Train Steps/Sec: 0.90 + 14%|██████████████████▉ | 6482/45917 [2:22:25<12:01:10, 1.10s/it][2025-04-24 12:42:08] (step=0052400) Train Loss: 6.3354, Train Steps/Sec: 0.89 + 14%|██████████████████▉ | 6507/45917 [2:23:00<26:50:20, 2.45s/it][2025-04-24 12:42:42] (step=0052425) Train Loss: 6.3398, Train Steps/Sec: 0.73 + 14%|███████████████████ | 6532/45917 [2:23:27<12:09:10, 1.11s/it][2025-04-24 12:43:10] (step=0052450) Train Loss: 6.3063, Train Steps/Sec: 0.90 + 14%|███████████████████▏ | 6557/45917 [2:24:01<13:14:30, 1.21s/it][2025-04-24 12:43:44] (step=0052475) Train Loss: 6.3423, Train Steps/Sec: 0.74 + 14%|███████████████████▏ | 6582/45917 [2:24:29<11:56:25, 1.09s/it][2025-04-24 12:44:11] (step=0052500) Train Loss: 6.3416, Train Steps/Sec: 0.90 + 14%|███████████████████▎ | 6607/45917 [2:24:57<12:18:29, 1.13s/it][2025-04-24 12:44:39] (step=0052525) Train Loss: 6.3980, Train Steps/Sec: 0.90 + 14%|███████████████████▎ | 6632/45917 [2:25:33<12:06:48, 1.11s/it][2025-04-24 12:45:15] (step=0052550) Train Loss: 6.3086, Train Steps/Sec: 0.70 + 14%|███████████████████▍ | 6657/45917 [2:26:01<12:00:41, 1.10s/it][2025-04-24 12:45:43] (step=0052575) Train Loss: 6.3095, Train Steps/Sec: 0.90 + 15%|███████████████████▌ | 6682/45917 [2:26:28<11:55:56, 1.09s/it][2025-04-24 12:46:11] (step=0052600) Train Loss: 6.3175, Train Steps/Sec: 0.90 + 15%|███████████████████▌ | 6707/45917 [2:26:56<12:17:54, 1.13s/it][2025-04-24 12:46:39] (step=0052625) Train Loss: 6.3343, Train Steps/Sec: 0.90 + 15%|███████████████████▋ | 6732/45917 [2:27:24<12:04:31, 1.11s/it][2025-04-24 12:47:06] (step=0052650) Train Loss: 6.2978, Train Steps/Sec: 0.90 + 15%|███████████████████▋ | 6757/45917 [2:27:52<11:51:20, 1.09s/it][2025-04-24 12:47:34] (step=0052675) Train Loss: 6.2779, Train Steps/Sec: 0.91 + 15%|███████████████████▊ | 6782/45917 [2:28:19<11:50:04, 1.09s/it][2025-04-24 12:48:02] (step=0052700) Train Loss: 6.2910, Train Steps/Sec: 0.90 + 15%|███████████████████▊ | 6807/45917 [2:28:47<12:15:26, 1.13s/it][2025-04-24 12:48:29] (step=0052725) Train Loss: 6.3636, Train Steps/Sec: 0.90 + 15%|███████████████████▉ | 6832/45917 [2:29:15<12:10:15, 1.12s/it][2025-04-24 12:48:57] (step=0052750) Train Loss: 6.2890, Train Steps/Sec: 0.90 + 15%|████████████████████ | 6857/45917 [2:29:43<11:56:25, 1.10s/it][2025-04-24 12:49:25] (step=0052775) Train Loss: 6.2934, Train Steps/Sec: 0.90 + 15%|████████████████████ | 6882/45917 [2:30:11<11:53:28, 1.10s/it][2025-04-24 12:49:53] (step=0052800) Train Loss: 6.2704, Train Steps/Sec: 0.90 + 15%|████████████████████▏ | 6907/45917 [2:30:39<12:13:43, 1.13s/it][2025-04-24 12:50:21] (step=0052825) Train Loss: 6.3100, Train Steps/Sec: 0.90 + 15%|████████████████████▏ | 6932/45917 [2:31:06<12:01:02, 1.11s/it][2025-04-24 12:50:48] (step=0052850) Train Loss: 6.3575, Train Steps/Sec: 0.90 + 15%|████████████████████▎ | 6957/45917 [2:31:34<11:54:41, 1.10s/it][2025-04-24 12:51:16] (step=0052875) Train Loss: 6.3414, Train Steps/Sec: 0.90 + 15%|████████████████████▍ | 6982/45917 [2:32:02<11:52:13, 1.10s/it][2025-04-24 12:51:44] (step=0052900) Train Loss: 6.3184, Train Steps/Sec: 0.90 + 15%|████████████████████▍ | 7007/45917 [2:32:30<12:16:37, 1.14s/it][2025-04-24 12:52:12] (step=0052925) Train Loss: 6.3065, Train Steps/Sec: 0.90 + 15%|████████████████████▌ | 7032/45917 [2:32:57<12:01:06, 1.11s/it][2025-04-24 12:52:39] (step=0052950) Train Loss: 6.2753, Train Steps/Sec: 0.90 + 15%|████████████████████▌ | 7057/45917 [2:33:25<11:56:12, 1.11s/it][2025-04-24 12:53:07] (step=0052975) Train Loss: 6.3487, Train Steps/Sec: 0.90 + 15%|████████████████████▋ | 7082/45917 [2:33:53<11:54:23, 1.10s/it][2025-04-24 12:53:35] (step=0053000) Train Loss: 6.3219, Train Steps/Sec: 0.90 + 15%|████████████████████▋ | 7107/45917 [2:34:21<12:09:25, 1.13s/it][2025-04-24 12:54:03] (step=0053025) Train Loss: 6.3145, Train Steps/Sec: 0.91 + 16%|████████████████████▊ | 7132/45917 [2:34:48<11:57:54, 1.11s/it][2025-04-24 12:54:30] (step=0053050) Train Loss: 6.3305, Train Steps/Sec: 0.90 + 16%|████████████████████▉ | 7157/45917 [2:35:16<11:51:40, 1.10s/it][2025-04-24 12:54:58] (step=0053075) Train Loss: 6.3119, Train Steps/Sec: 0.90 + 16%|████████████████████▉ | 7182/45917 [2:35:44<11:55:28, 1.11s/it][2025-04-24 12:55:26] (step=0053100) Train Loss: 6.2969, Train Steps/Sec: 0.89 + 16%|█████████████████████ | 7207/45917 [2:36:12<12:02:55, 1.12s/it][2025-04-24 12:55:54] (step=0053125) Train Loss: 6.3449, Train Steps/Sec: 0.90 + 16%|█████████████████████ | 7232/45917 [2:36:40<11:55:03, 1.11s/it][2025-04-24 12:56:22] (step=0053150) Train Loss: 6.3931, Train Steps/Sec: 0.90 + 16%|█████████████████████▏ | 7257/45917 [2:37:07<11:48:12, 1.10s/it][2025-04-24 12:56:49] (step=0053175) Train Loss: 6.2805, Train Steps/Sec: 0.90 + 16%|█████████████████████▎ | 7282/45917 [2:37:35<11:47:46, 1.10s/it][2025-04-24 12:57:17] (step=0053200) Train Loss: 6.2878, Train Steps/Sec: 0.90 + 16%|█████████████████████▎ | 7307/45917 [2:38:03<12:05:02, 1.13s/it][2025-04-24 12:57:45] (step=0053225) Train Loss: 6.3526, Train Steps/Sec: 0.90 + 16%|█████████████████████▍ | 7332/45917 [2:38:31<11:51:59, 1.11s/it][2025-04-24 12:58:13] (step=0053250) Train Loss: 6.3640, Train Steps/Sec: 0.90 + 16%|█████████████████████▍ | 7357/45917 [2:38:58<11:53:09, 1.11s/it][2025-04-24 12:58:40] (step=0053275) Train Loss: 6.3440, Train Steps/Sec: 0.90 + 16%|█████████████████████▌ | 7382/45917 [2:39:26<11:39:44, 1.09s/it][2025-04-24 12:59:08] (step=0053300) Train Loss: 6.3444, Train Steps/Sec: 0.90 + 16%|█████████████████████▌ | 7407/45917 [2:39:54<12:03:50, 1.13s/it][2025-04-24 12:59:36] (step=0053325) Train Loss: 6.3591, Train Steps/Sec: 0.90 + 16%|█████████████████████▋ | 7432/45917 [2:40:22<11:53:39, 1.11s/it][2025-04-24 13:00:04] (step=0053350) Train Loss: 6.3428, Train Steps/Sec: 0.90 + 16%|█████████████████████▊ | 7457/45917 [2:40:49<11:49:15, 1.11s/it][2025-04-24 13:00:32] (step=0053375) Train Loss: 6.3343, Train Steps/Sec: 0.90 + 16%|█████████████████████▊ | 7482/45917 [2:41:17<11:44:19, 1.10s/it][2025-04-24 13:01:00] (step=0053400) Train Loss: 6.3173, Train Steps/Sec: 0.89 + 16%|█████████████████████▉ | 7507/45917 [2:41:45<12:02:05, 1.13s/it][2025-04-24 13:01:27] (step=0053425) Train Loss: 6.3290, Train Steps/Sec: 0.90 + 16%|█████████████████████▉ | 7532/45917 [2:42:13<11:49:06, 1.11s/it][2025-04-24 13:01:55] (step=0053450) Train Loss: 6.3461, Train Steps/Sec: 0.90 + 16%|██████████████████████ | 7557/45917 [2:42:41<11:43:28, 1.10s/it][2025-04-24 13:02:23] (step=0053475) Train Loss: 6.2877, Train Steps/Sec: 0.90 + 17%|██████████████████████▏ | 7582/45917 [2:43:08<11:46:23, 1.11s/it][2025-04-24 13:02:51] (step=0053500) Train Loss: 6.2885, Train Steps/Sec: 0.90 + 17%|██████████████████████▏ | 7607/45917 [2:43:36<11:55:02, 1.12s/it][2025-04-24 13:03:18] (step=0053525) Train Loss: 6.2769, Train Steps/Sec: 0.90 + 17%|██████████████████████▎ | 7632/45917 [2:44:04<11:50:51, 1.11s/it][2025-04-24 13:03:46] (step=0053550) Train Loss: 6.3005, Train Steps/Sec: 0.90 + 17%|██████████████████████▎ | 7657/45917 [2:44:32<11:43:23, 1.10s/it][2025-04-24 13:04:14] (step=0053575) Train Loss: 6.3316, Train Steps/Sec: 0.90 + 17%|██████████████████████▍ | 7682/45917 [2:44:59<11:40:17, 1.10s/it][2025-04-24 13:04:42] (step=0053600) Train Loss: 6.3100, Train Steps/Sec: 0.90 + 17%|██████████████████████▍ | 7707/45917 [2:45:27<12:08:35, 1.14s/it][2025-04-24 13:05:09] (step=0053625) Train Loss: 6.3550, Train Steps/Sec: 0.90 + 17%|██████████████████████▌ | 7732/45917 [2:45:55<11:43:58, 1.11s/it][2025-04-24 13:05:37] (step=0053650) Train Loss: 6.3134, Train Steps/Sec: 0.90 + 17%|██████████████████████▋ | 7757/45917 [2:46:23<11:41:57, 1.10s/it][2025-04-24 13:06:05] (step=0053675) Train Loss: 6.3506, Train Steps/Sec: 0.90 + 17%|██████████████████████▋ | 7782/45917 [2:46:51<11:34:15, 1.09s/it][2025-04-24 13:06:33] (step=0053700) Train Loss: 6.3241, Train Steps/Sec: 0.90 + 17%|██████████████████████▊ | 7807/45917 [2:47:18<11:57:37, 1.13s/it][2025-04-24 13:07:00] (step=0053725) Train Loss: 6.3265, Train Steps/Sec: 0.90 + 17%|██████████████████████▊ | 7832/45917 [2:47:46<11:42:25, 1.11s/it][2025-04-24 13:07:28] (step=0053750) Train Loss: 6.2896, Train Steps/Sec: 0.90 + 17%|██████████████████████▉ | 7857/45917 [2:48:14<11:42:45, 1.11s/it][2025-04-24 13:07:56] (step=0053775) Train Loss: 6.3623, Train Steps/Sec: 0.90 + 17%|███████████████████████ | 7882/45917 [2:48:42<11:34:49, 1.10s/it][2025-04-24 13:08:24] (step=0053800) Train Loss: 6.3295, Train Steps/Sec: 0.90 + 17%|███████████████████████ | 7907/45917 [2:49:09<11:57:08, 1.13s/it][2025-04-24 13:08:52] (step=0053825) Train Loss: 6.3478, Train Steps/Sec: 0.90 + 17%|███████████████████████▏ | 7932/45917 [2:49:37<11:38:25, 1.10s/it][2025-04-24 13:09:19] (step=0053850) Train Loss: 6.3172, Train Steps/Sec: 0.90 + 17%|███████████████████████▏ | 7957/45917 [2:50:05<11:36:20, 1.10s/it][2025-04-24 13:09:47] (step=0053875) Train Loss: 6.2940, Train Steps/Sec: 0.90 + 17%|███████████████████████▎ | 7982/45917 [2:50:33<11:39:39, 1.11s/it][2025-04-24 13:10:15] (step=0053900) Train Loss: 6.3266, Train Steps/Sec: 0.90 + 17%|███████████████████████▎ | 8007/45917 [2:51:01<12:04:12, 1.15s/it][2025-04-24 13:10:43] (step=0053925) Train Loss: 6.3277, Train Steps/Sec: 0.90 + 17%|███████████████████████▍ | 8032/45917 [2:51:28<11:49:33, 1.12s/it][2025-04-24 13:11:11] (step=0053950) Train Loss: 6.2985, Train Steps/Sec: 0.90 + 18%|███████████████████████▌ | 8057/45917 [2:51:56<11:37:33, 1.11s/it][2025-04-24 13:11:38] (step=0053975) Train Loss: 6.2941, Train Steps/Sec: 0.90 + 18%|███████████████████████▌ | 8082/45917 [2:52:24<11:34:03, 1.10s/it][2025-04-24 13:12:06] (step=0054000) Train Loss: 6.3787, Train Steps/Sec: 0.90 +[2025-04-24 13:12:06] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 13:12:06] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 13:12:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.81s/it] +[2025-04-24 13:16:53] Finish Eval in 54000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.31s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 13:17:15] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0054000.pt +[2025-04-24 13:17:17] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0052000.pt + 18%|███████████████████████▋ | 8107/45917 [2:58:03<12:01:19, 1.14s/it][2025-04-24 13:17:46] (step=0054025) Train Loss: 6.3089, Train Steps/Sec: 0.07 + 18%|███████████████████████▋ | 8132/45917 [2:58:31<11:44:23, 1.12s/it][2025-04-24 13:18:13] (step=0054050) Train Loss: 6.3055, Train Steps/Sec: 0.90 + 18%|███████████████████████▊ | 8157/45917 [2:58:59<11:33:00, 1.10s/it][2025-04-24 13:18:41] (step=0054075) Train Loss: 6.3260, Train Steps/Sec: 0.90 + 18%|███████████████████████▉ | 8182/45917 [2:59:27<11:35:09, 1.11s/it][2025-04-24 13:19:09] (step=0054100) Train Loss: 6.3198, Train Steps/Sec: 0.90 + 18%|███████████████████████▉ | 8207/45917 [2:59:55<11:49:13, 1.13s/it][2025-04-24 13:19:37] (step=0054125) Train Loss: 6.3209, Train Steps/Sec: 0.90 + 18%|████████████████████████ | 8232/45917 [3:00:22<11:38:02, 1.11s/it][2025-04-24 13:20:04] (step=0054150) Train Loss: 6.3413, Train Steps/Sec: 0.90 + 18%|████████████████████████ | 8257/45917 [3:00:50<11:28:43, 1.10s/it][2025-04-24 13:20:32] (step=0054175) Train Loss: 6.2346, Train Steps/Sec: 0.91 + 18%|████████████████████████▏ | 8282/45917 [3:01:18<11:31:43, 1.10s/it][2025-04-24 13:21:00] (step=0054200) Train Loss: 6.3840, Train Steps/Sec: 0.88 + 18%|████████████████████████▏ | 8307/45917 [3:01:46<11:48:21, 1.13s/it][2025-04-24 13:21:28] (step=0054225) Train Loss: 6.2912, Train Steps/Sec: 0.90 + 18%|████████████████████████▎ | 8332/45917 [3:02:13<11:29:11, 1.10s/it][2025-04-24 13:21:56] (step=0054250) Train Loss: 6.3325, Train Steps/Sec: 0.91 + 18%|████████████████████████▍ | 8357/45917 [3:02:41<11:28:32, 1.10s/it][2025-04-24 13:22:23] (step=0054275) Train Loss: 6.3134, Train Steps/Sec: 0.90 + 18%|████████████████████████▍ | 8382/45917 [3:03:09<11:28:04, 1.10s/it][2025-04-24 13:22:51] (step=0054300) Train Loss: 6.3682, Train Steps/Sec: 0.89 + 18%|████████████████████████▌ | 8407/45917 [3:03:37<11:37:57, 1.12s/it][2025-04-24 13:23:19] (step=0054325) Train Loss: 6.2786, Train Steps/Sec: 0.91 + 18%|████████████████████████▌ | 8432/45917 [3:04:04<11:35:11, 1.11s/it][2025-04-24 13:23:47] (step=0054350) Train Loss: 6.2993, Train Steps/Sec: 0.90 + 18%|████████████████████████▋ | 8457/45917 [3:04:32<11:34:56, 1.11s/it][2025-04-24 13:24:14] (step=0054375) Train Loss: 6.3273, Train Steps/Sec: 0.90 + 18%|████████████████████████▊ | 8482/45917 [3:05:00<11:28:00, 1.10s/it][2025-04-24 13:24:42] (step=0054400) Train Loss: 6.3054, Train Steps/Sec: 0.90 + 19%|████████████████████████▊ | 8507/45917 [3:05:28<11:44:43, 1.13s/it][2025-04-24 13:25:10] (step=0054425) Train Loss: 6.3380, Train Steps/Sec: 0.90 + 19%|████████████████████████▉ | 8532/45917 [3:05:56<11:31:11, 1.11s/it][2025-04-24 13:25:38] (step=0054450) Train Loss: 6.2943, Train Steps/Sec: 0.90 + 19%|████████████████████████▉ | 8557/45917 [3:06:23<11:25:37, 1.10s/it][2025-04-24 13:26:05] (step=0054475) Train Loss: 6.3285, Train Steps/Sec: 0.90 + 19%|█████████████████████████ | 8582/45917 [3:06:51<11:25:05, 1.10s/it][2025-04-24 13:26:33] (step=0054500) Train Loss: 6.2794, Train Steps/Sec: 0.90 + 19%|█████████████████████████ | 8607/45917 [3:07:19<11:42:57, 1.13s/it][2025-04-24 13:27:01] (step=0054525) Train Loss: 6.3150, Train Steps/Sec: 0.90 + 19%|█████████████████████████▏ | 8632/45917 [3:07:47<11:32:15, 1.11s/it][2025-04-24 13:27:29] (step=0054550) Train Loss: 6.3435, Train Steps/Sec: 0.90 + 19%|█████████████████████████▎ | 8657/45917 [3:08:14<11:28:02, 1.11s/it][2025-04-24 13:27:56] (step=0054575) Train Loss: 6.3056, Train Steps/Sec: 0.90 + 19%|█████████████████████████▎ | 8682/45917 [3:08:42<11:18:24, 1.09s/it][2025-04-24 13:28:24] (step=0054600) Train Loss: 6.2695, Train Steps/Sec: 0.90 + 19%|█████████████████████████▍ | 8707/45917 [3:09:10<11:39:57, 1.13s/it][2025-04-24 13:28:52] (step=0054625) Train Loss: 6.3249, Train Steps/Sec: 0.90 + 19%|█████████████████████████▍ | 8732/45917 [3:09:37<11:24:33, 1.10s/it][2025-04-24 13:29:20] (step=0054650) Train Loss: 6.3349, Train Steps/Sec: 0.90 + 19%|█████████████████████████▌ | 8757/45917 [3:10:05<11:23:28, 1.10s/it][2025-04-24 13:29:47] (step=0054675) Train Loss: 6.3151, Train Steps/Sec: 0.90 + 19%|█████████████████████████▋ | 8782/45917 [3:10:33<11:19:49, 1.10s/it][2025-04-24 13:30:15] (step=0054700) Train Loss: 6.2762, Train Steps/Sec: 0.89 + 19%|█████████████████████████▋ | 8807/45917 [3:11:01<11:40:17, 1.13s/it][2025-04-24 13:30:43] (step=0054725) Train Loss: 6.3408, Train Steps/Sec: 0.90 + 19%|█████████████████████████▊ | 8832/45917 [3:11:28<11:24:43, 1.11s/it][2025-04-24 13:31:11] (step=0054750) Train Loss: 6.2837, Train Steps/Sec: 0.90 + 19%|█████████████████████████▊ | 8857/45917 [3:11:56<11:22:18, 1.10s/it][2025-04-24 13:31:38] (step=0054775) Train Loss: 6.3121, Train Steps/Sec: 0.90 + 19%|█████████████████████████▉ | 8882/45917 [3:12:24<11:18:19, 1.10s/it][2025-04-24 13:32:06] (step=0054800) Train Loss: 6.2816, Train Steps/Sec: 0.90 + 19%|█████████████████████████▉ | 8907/45917 [3:12:52<11:33:52, 1.12s/it][2025-04-24 13:32:34] (step=0054825) Train Loss: 6.2911, Train Steps/Sec: 0.90 + 19%|██████████████████████████ | 8932/45917 [3:13:19<11:26:43, 1.11s/it][2025-04-24 13:33:02] (step=0054850) Train Loss: 6.3060, Train Steps/Sec: 0.90 + 20%|██████████████████████████▏ | 8957/45917 [3:13:47<11:20:28, 1.10s/it][2025-04-24 13:33:29] (step=0054875) Train Loss: 6.3172, Train Steps/Sec: 0.90 + 20%|██████████████████████████▏ | 8982/45917 [3:14:15<11:21:32, 1.11s/it][2025-04-24 13:33:57] (step=0054900) Train Loss: 6.3269, Train Steps/Sec: 0.89 + 20%|██████████████████████████▎ | 9007/45917 [3:14:43<11:26:06, 1.12s/it][2025-04-24 13:34:25] (step=0054925) Train Loss: 6.3182, Train Steps/Sec: 0.90 + 20%|██████████████████████████▎ | 9032/45917 [3:15:11<11:21:43, 1.11s/it][2025-04-24 13:34:53] (step=0054950) Train Loss: 6.3596, Train Steps/Sec: 0.90 + 20%|██████████████████████████▍ | 9057/45917 [3:15:38<11:17:03, 1.10s/it][2025-04-24 13:35:21] (step=0054975) Train Loss: 6.3326, Train Steps/Sec: 0.89 + 20%|██████████████████████████▌ | 9082/45917 [3:16:06<11:13:09, 1.10s/it][2025-04-24 13:35:48] (step=0055000) Train Loss: 6.3418, Train Steps/Sec: 0.90 + 20%|██████████████████████████▌ | 9107/45917 [3:16:34<11:36:43, 1.14s/it][2025-04-24 13:36:16] (step=0055025) Train Loss: 6.3343, Train Steps/Sec: 0.90 + 20%|██████████████████████████▋ | 9132/45917 [3:17:02<11:24:01, 1.12s/it][2025-04-24 13:36:44] (step=0055050) Train Loss: 6.3875, Train Steps/Sec: 0.90 + 20%|██████████████████████████▋ | 9157/45917 [3:17:30<11:21:19, 1.11s/it][2025-04-24 13:37:12] (step=0055075) Train Loss: 6.3424, Train Steps/Sec: 0.90 + 20%|██████████████████████████▊ | 9182/45917 [3:17:57<11:22:11, 1.11s/it][2025-04-24 13:37:40] (step=0055100) Train Loss: 6.3391, Train Steps/Sec: 0.90 + 20%|██████████████████████████▊ | 9207/45917 [3:18:32<11:48:01, 1.16s/it][2025-04-24 13:38:14] (step=0055125) Train Loss: 6.3566, Train Steps/Sec: 0.72 + 20%|██████████████████████████▉ | 9232/45917 [3:19:00<11:23:02, 1.12s/it][2025-04-24 13:38:42] (step=0055150) Train Loss: 6.3223, Train Steps/Sec: 0.90 + 20%|███████████████████████████ | 9257/45917 [3:19:35<11:33:13, 1.13s/it][2025-04-24 13:39:17] (step=0055175) Train Loss: 6.3628, Train Steps/Sec: 0.72 + 20%|███████████████████████████ | 9282/45917 [3:20:02<11:09:53, 1.10s/it][2025-04-24 13:39:45] (step=0055200) Train Loss: 6.3278, Train Steps/Sec: 0.90 + 20%|███████████████████████████▏ | 9307/45917 [3:20:30<11:28:13, 1.13s/it][2025-04-24 13:40:12] (step=0055225) Train Loss: 6.2815, Train Steps/Sec: 0.90 + 20%|███████████████████████████▏ | 9332/45917 [3:20:58<11:19:13, 1.11s/it][2025-04-24 13:40:40] (step=0055250) Train Loss: 6.3367, Train Steps/Sec: 0.90 + 20%|███████████████████████████▎ | 9357/45917 [3:21:26<11:13:11, 1.10s/it][2025-04-24 13:41:08] (step=0055275) Train Loss: 6.3064, Train Steps/Sec: 0.90 + 20%|███████████████████████████▍ | 9382/45917 [3:21:54<11:12:53, 1.11s/it][2025-04-24 13:41:36] (step=0055300) Train Loss: 6.3465, Train Steps/Sec: 0.90 + 20%|███████████████████████████▍ | 9407/45917 [3:22:21<11:27:38, 1.13s/it][2025-04-24 13:42:04] (step=0055325) Train Loss: 6.3079, Train Steps/Sec: 0.90 + 21%|███████████████████████████▌ | 9432/45917 [3:22:49<11:15:37, 1.11s/it][2025-04-24 13:42:31] (step=0055350) Train Loss: 6.3465, Train Steps/Sec: 0.90 + 21%|███████████████████████████▌ | 9457/45917 [3:23:17<11:13:03, 1.11s/it][2025-04-24 13:42:59] (step=0055375) Train Loss: 6.2833, Train Steps/Sec: 0.90 + 21%|███████████████████████████▋ | 9482/45917 [3:23:45<11:01:54, 1.09s/it][2025-04-24 13:43:27] (step=0055400) Train Loss: 6.2832, Train Steps/Sec: 0.90 + 21%|███████████████████████████▋ | 9507/45917 [3:24:13<11:28:41, 1.13s/it][2025-04-24 13:43:55] (step=0055425) Train Loss: 6.3137, Train Steps/Sec: 0.90 + 21%|███████████████████████████▊ | 9532/45917 [3:24:40<11:15:39, 1.11s/it][2025-04-24 13:44:23] (step=0055450) Train Loss: 6.3506, Train Steps/Sec: 0.90 + 21%|███████████████████████████▉ | 9557/45917 [3:25:08<11:09:48, 1.11s/it][2025-04-24 13:44:50] (step=0055475) Train Loss: 6.3098, Train Steps/Sec: 0.90 + 21%|███████████████████████████▉ | 9582/45917 [3:25:36<11:07:50, 1.10s/it][2025-04-24 13:45:18] (step=0055500) Train Loss: 6.3374, Train Steps/Sec: 0.90 + 21%|████████████████████████████ | 9607/45917 [3:26:04<11:24:45, 1.13s/it][2025-04-24 13:45:46] (step=0055525) Train Loss: 6.3489, Train Steps/Sec: 0.90 + 21%|████████████████████████████ | 9632/45917 [3:26:31<11:10:49, 1.11s/it][2025-04-24 13:46:14] (step=0055550) Train Loss: 6.3443, Train Steps/Sec: 0.90 + 21%|████████████████████████████▏ | 9657/45917 [3:26:59<11:12:34, 1.11s/it][2025-04-24 13:46:41] (step=0055575) Train Loss: 6.3137, Train Steps/Sec: 0.90 + 21%|████████████████████████████▎ | 9682/45917 [3:27:27<10:58:48, 1.09s/it][2025-04-24 13:47:09] (step=0055600) Train Loss: 6.3346, Train Steps/Sec: 0.90 + 21%|████████████████████████████▎ | 9707/45917 [3:27:55<11:20:58, 1.13s/it][2025-04-24 13:47:37] (step=0055625) Train Loss: 6.2865, Train Steps/Sec: 0.91 + 21%|████████████████████████████▍ | 9732/45917 [3:28:22<11:09:37, 1.11s/it][2025-04-24 13:48:05] (step=0055650) Train Loss: 6.3210, Train Steps/Sec: 0.90 + 21%|████████████████████████████▍ | 9757/45917 [3:28:50<11:09:59, 1.11s/it][2025-04-24 13:48:32] (step=0055675) Train Loss: 6.3549, Train Steps/Sec: 0.90 + 21%|████████████████████████████▌ | 9782/45917 [3:29:18<11:01:44, 1.10s/it][2025-04-24 13:49:00] (step=0055700) Train Loss: 6.3318, Train Steps/Sec: 0.90 + 21%|████████████████████████████▌ | 9807/45917 [3:29:46<11:16:50, 1.12s/it][2025-04-24 13:49:28] (step=0055725) Train Loss: 6.3322, Train Steps/Sec: 0.90 + 21%|████████████████████████████▋ | 9832/45917 [3:30:14<11:07:29, 1.11s/it][2025-04-24 13:49:56] (step=0055750) Train Loss: 6.3017, Train Steps/Sec: 0.90 + 21%|████████████████████████████▊ | 9857/45917 [3:30:41<11:04:16, 1.11s/it][2025-04-24 13:50:24] (step=0055775) Train Loss: 6.3214, Train Steps/Sec: 0.90 + 22%|████████████████████████████▊ | 9882/45917 [3:31:09<11:05:55, 1.11s/it][2025-04-24 13:50:51] (step=0055800) Train Loss: 6.3864, Train Steps/Sec: 0.89 + 22%|████████████████████████████▉ | 9907/45917 [3:31:37<11:23:43, 1.14s/it][2025-04-24 13:51:19] (step=0055825) Train Loss: 6.3401, Train Steps/Sec: 0.90 + 22%|████████████████████████████▉ | 9932/45917 [3:32:05<11:08:35, 1.11s/it][2025-04-24 13:51:47] (step=0055850) Train Loss: 6.2594, Train Steps/Sec: 0.90 + 22%|█████████████████████████████ | 9957/45917 [3:32:33<11:01:33, 1.10s/it][2025-04-24 13:52:15] (step=0055875) Train Loss: 6.3620, Train Steps/Sec: 0.90 + 22%|█████████████████████████████▏ | 9982/45917 [3:33:01<11:01:17, 1.10s/it][2025-04-24 13:52:43] (step=0055900) Train Loss: 6.3331, Train Steps/Sec: 0.89 + 22%|████████████████████████████▉ | 10007/45917 [3:33:28<11:12:32, 1.12s/it][2025-04-24 13:53:11] (step=0055925) Train Loss: 6.3208, Train Steps/Sec: 0.91 + 22%|█████████████████████████████ | 10032/45917 [3:33:56<11:01:32, 1.11s/it][2025-04-24 13:53:38] (step=0055950) Train Loss: 6.3214, Train Steps/Sec: 0.90 + 22%|█████████████████████████████▏ | 10057/45917 [3:34:24<10:58:34, 1.10s/it][2025-04-24 13:54:06] (step=0055975) Train Loss: 6.3121, Train Steps/Sec: 0.90 + 22%|█████████████████████████████▏ | 10082/45917 [3:34:52<10:55:52, 1.10s/it][2025-04-24 13:54:34] (step=0056000) Train Loss: 6.3536, Train Steps/Sec: 0.90 +[2025-04-24 13:54:34] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 13:54:34] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 13:54:34] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.98s/it] +[2025-04-24 13:59:21] Finish Eval in 56000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.47s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 13:59:43] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0056000.pt +[2025-04-24 13:59:45] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0054000.pt + 22%|█████████████████████████████▎ | 10107/45917 [3:40:31<11:23:02, 1.14s/it][2025-04-24 14:00:14] (step=0056025) Train Loss: 6.2973, Train Steps/Sec: 0.07 + 22%|█████████████████████████████▎ | 10132/45917 [3:41:00<11:29:15, 1.16s/it][2025-04-24 14:00:42] (step=0056050) Train Loss: 6.3303, Train Steps/Sec: 0.89 + 22%|█████████████████████████████▍ | 10157/45917 [3:41:27<11:00:58, 1.11s/it][2025-04-24 14:01:10] (step=0056075) Train Loss: 6.3107, Train Steps/Sec: 0.90 + 22%|█████████████████████████████▍ | 10182/45917 [3:41:55<10:55:22, 1.10s/it][2025-04-24 14:01:37] (step=0056100) Train Loss: 6.2994, Train Steps/Sec: 0.90 + 22%|█████████████████████████████▌ | 10207/45917 [3:42:23<11:10:28, 1.13s/it][2025-04-24 14:02:05] (step=0056125) Train Loss: 6.3344, Train Steps/Sec: 0.90 + 22%|█████████████████████████████▋ | 10232/45917 [3:42:51<11:03:52, 1.12s/it][2025-04-24 14:02:33] (step=0056150) Train Loss: 6.3162, Train Steps/Sec: 0.90 + 22%|█████████████████████████████▋ | 10257/45917 [3:43:19<11:02:30, 1.11s/it][2025-04-24 14:03:01] (step=0056175) Train Loss: 6.3133, Train Steps/Sec: 0.90 + 22%|█████████████████████████████▊ | 10282/45917 [3:43:46<11:00:45, 1.11s/it][2025-04-24 14:03:29] (step=0056200) Train Loss: 6.3107, Train Steps/Sec: 0.90 + 22%|█████████████████████████████▊ | 10307/45917 [3:44:14<11:23:24, 1.15s/it][2025-04-24 14:03:56] (step=0056225) Train Loss: 6.3266, Train Steps/Sec: 0.90 + 23%|█████████████████████████████▉ | 10332/45917 [3:44:42<10:58:51, 1.11s/it][2025-04-24 14:04:24] (step=0056250) Train Loss: 6.3309, Train Steps/Sec: 0.90 + 23%|█████████████████████████████▉ | 10357/45917 [3:45:10<10:51:13, 1.10s/it][2025-04-24 14:04:52] (step=0056275) Train Loss: 6.3255, Train Steps/Sec: 0.90 + 23%|██████████████████████████████ | 10382/45917 [3:45:37<10:54:15, 1.10s/it][2025-04-24 14:05:20] (step=0056300) Train Loss: 6.3762, Train Steps/Sec: 0.90 + 23%|██████████████████████████████▏ | 10407/45917 [3:46:05<11:08:46, 1.13s/it][2025-04-24 14:05:47] (step=0056325) Train Loss: 6.3270, Train Steps/Sec: 0.90 + 23%|██████████████████████████████▏ | 10432/45917 [3:46:33<10:51:38, 1.10s/it][2025-04-24 14:06:15] (step=0056350) Train Loss: 6.3681, Train Steps/Sec: 0.91 + 23%|██████████████████████████████▎ | 10457/45917 [3:47:01<10:50:13, 1.10s/it][2025-04-24 14:06:43] (step=0056375) Train Loss: 6.3292, Train Steps/Sec: 0.90 + 23%|██████████████████████████████▎ | 10482/45917 [3:47:28<10:47:44, 1.10s/it][2025-04-24 14:07:11] (step=0056400) Train Loss: 6.3453, Train Steps/Sec: 0.89 + 23%|██████████████████████████████▍ | 10507/45917 [3:47:56<11:04:12, 1.13s/it][2025-04-24 14:07:38] (step=0056425) Train Loss: 6.3141, Train Steps/Sec: 0.90 + 23%|██████████████████████████████▌ | 10532/45917 [3:48:24<10:58:19, 1.12s/it][2025-04-24 14:08:06] (step=0056450) Train Loss: 6.3105, Train Steps/Sec: 0.90 + 23%|██████████████████████████████▌ | 10557/45917 [3:48:52<10:43:24, 1.09s/it][2025-04-24 14:08:34] (step=0056475) Train Loss: 6.3115, Train Steps/Sec: 0.90 + 23%|██████████████████████████████▋ | 10582/45917 [3:49:19<10:48:30, 1.10s/it][2025-04-24 14:09:02] (step=0056500) Train Loss: 6.3080, Train Steps/Sec: 0.90 + 23%|██████████████████████████████▋ | 10607/45917 [3:49:47<11:01:03, 1.12s/it][2025-04-24 14:09:29] (step=0056525) Train Loss: 6.3562, Train Steps/Sec: 0.90 + 23%|██████████████████████████████▊ | 10632/45917 [3:50:15<10:57:26, 1.12s/it][2025-04-24 14:09:57] (step=0056550) Train Loss: 6.3494, Train Steps/Sec: 0.90 + 23%|██████████████████████████████▊ | 10657/45917 [3:50:43<10:48:37, 1.10s/it][2025-04-24 14:10:25] (step=0056575) Train Loss: 6.3319, Train Steps/Sec: 0.90 + 23%|██████████████████████████████▉ | 10682/45917 [3:51:11<10:39:47, 1.09s/it][2025-04-24 14:10:53] (step=0056600) Train Loss: 6.2593, Train Steps/Sec: 0.90 + 23%|███████████████████████████████ | 10707/45917 [3:51:38<11:01:47, 1.13s/it][2025-04-24 14:11:21] (step=0056625) Train Loss: 6.3549, Train Steps/Sec: 0.90 + 23%|███████████████████████████████ | 10732/45917 [3:52:06<10:47:52, 1.10s/it][2025-04-24 14:11:48] (step=0056650) Train Loss: 6.2594, Train Steps/Sec: 0.90 + 23%|███████████████████████████████▏ | 10757/45917 [3:52:34<10:44:49, 1.10s/it][2025-04-24 14:12:16] (step=0056675) Train Loss: 6.2747, Train Steps/Sec: 0.90 + 23%|███████████████████████████████▏ | 10782/45917 [3:53:02<10:44:20, 1.10s/it][2025-04-24 14:12:44] (step=0056700) Train Loss: 6.2900, Train Steps/Sec: 0.90 + 24%|███████████████████████████████▎ | 10807/45917 [3:53:29<10:56:55, 1.12s/it][2025-04-24 14:13:12] (step=0056725) Train Loss: 6.3382, Train Steps/Sec: 0.90 + 24%|███████████████████████████████▍ | 10832/45917 [3:53:57<10:48:29, 1.11s/it][2025-04-24 14:13:39] (step=0056750) Train Loss: 6.2685, Train Steps/Sec: 0.90 + 24%|███████████████████████████████▍ | 10857/45917 [3:54:25<10:39:42, 1.09s/it][2025-04-24 14:14:07] (step=0056775) Train Loss: 6.3690, Train Steps/Sec: 0.90 + 24%|███████████████████████████████▌ | 10882/45917 [3:54:53<10:36:46, 1.09s/it][2025-04-24 14:14:35] (step=0056800) Train Loss: 6.3079, Train Steps/Sec: 0.90 + 24%|███████████████████████████████▌ | 10907/45917 [3:55:20<11:06:27, 1.14s/it][2025-04-24 14:15:03] (step=0056825) Train Loss: 6.2733, Train Steps/Sec: 0.90 + 24%|███████████████████████████████▋ | 10932/45917 [3:55:48<10:48:25, 1.11s/it][2025-04-24 14:15:30] (step=0056850) Train Loss: 6.3278, Train Steps/Sec: 0.90 + 24%|███████████████████████████████▋ | 10957/45917 [3:56:16<10:38:51, 1.10s/it][2025-04-24 14:15:58] (step=0056875) Train Loss: 6.3499, Train Steps/Sec: 0.90 + 24%|███████████████████████████████▊ | 10963/45917 [3:56:23<10:57:30, 1.13s/it] diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/files/requirements.txt b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8d52c600fa38ccbfa2604fb984f24a6dbb7afdb1 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/files/requirements.txt @@ -0,0 +1,131 @@ +setproctitle==1.3.5 +nvidia-nvjitlink-cu12==12.1.105 +nvidia-nvtx-cu12==12.1.105 +debugpy==1.8.13 +pyarrow==19.0.1 +click==8.1.8 +tqdm==4.67.1 +decorator==5.2.1 +cycler==0.12.1 +pydantic==2.10.6 +jedi==0.19.2 +PyYAML==6.0.2 +torchaudio==2.5.1+cu121 +wheel==0.45.1 +pure_eval==0.2.3 +nvidia-cudnn-cu12==9.1.0.70 +certifi==2025.1.31 +pillow==11.1.0 +typing_extensions==4.12.2 +nvitop==1.4.2 +nvidia-curand-cu12==10.3.2.106 +pyparsing==3.2.1 +docker-pycreds==0.4.0 +frozenlist==1.5.0 +wcwidth==0.2.13 +platformdirs==4.3.6 +jupyter_core==5.7.2 +kiwisolver==1.4.8 +prompt_toolkit==3.0.50 +datasets==3.4.1 +comm==0.2.2 +joblib==1.4.2 +pip==25.0 +transformers==4.49.0 +sentry-sdk==2.23.1 +aiohttp==3.11.14 +nest_asyncio==1.6.0 +triton==3.1.0 +Jinja2==3.1.4 +multiprocess==0.70.16 +fsspec==2024.12.0 +executing==2.2.0 +executing==2.1.0 +nvidia-cuda-cupti-cu12==12.1.105 +attrs==25.3.0 +GitPython==3.1.44 +ptyprocess==0.7.0 +matplotlib-inline==0.1.7 +pydantic_core==2.27.2 +pexpect==4.9.0 +urllib3==2.3.0 +psutil==7.0.0 +jupyter_client==8.6.3 +dill==0.3.8 +wandb==0.19.8 +ipykernel==6.29.5 +aiosignal==1.3.2 +contourpy==1.3.1 +propcache==0.3.0 +networkx==3.3 +setuptools==75.8.0 +nvidia-cusolver-cu12==11.4.5.107 +packaging==24.2 +nltk==3.9.1 +fonttools==4.56.0 +nvidia-ml-py==12.570.86 +smmap==5.0.2 +nvidia-cusparse-cu12==12.1.0.106 +filelock==3.18.0 +nvidia-cublas-cu12==12.1.3.1 +Pygments==2.19.1 +pandas==2.2.3 +torchvision==0.20.1+cu121 +annotated-types==0.7.0 +traitlets==5.14.3 +yarl==1.18.3 +sympy==1.13.1 +pickleshare==0.7.5 +xxhash==3.5.0 +exceptiongroup==1.2.2 +matplotlib==3.10.1 +stack_data==0.6.3 +pytz==2025.1 +protobuf==5.29.3 +nvidia-nccl-cu12==2.21.5 +regex==2024.11.6 +multidict==6.2.0 +MarkupSafe==2.1.5 +gitdb==4.0.12 +nvidia-cufft-cu12==11.0.2.54 +charset-normalizer==3.4.1 +opencv-python==4.11.0.86 +safetensors==0.5.3 +ipython==9.0.2 +asttokens==3.0.0 +ipython_pygments_lexers==1.1.1 +parso==0.8.4 +idna==3.10 +python-dateutil==2.9.0.post0 +tornado==6.4.2 +numpy==2.2.4 +pyzmq==26.3.0 +hf_transfer==0.1.9 +torch==2.5.1+cu121 +nvidia-cuda-nvrtc-cu12==12.1.105 +zipp==3.21.0 +mpmath==1.3.0 +aiohappyeyeballs==2.6.1 +six==1.17.0 +huggingface-hub==0.29.3 +requests==2.32.3 +tzdata==2025.1 +tokenizers==0.21.1 +importlib_metadata==8.6.1 +nvidia-cuda-runtime-cu12==12.1.105 +jaraco.functools==4.0.1 +typeguard==4.3.0 +typing_extensions==4.12.2 +wheel==0.43.0 +platformdirs==4.2.2 +jaraco.context==5.3.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +jaraco.collections==5.1.0 +autocommand==2.2.2 +zipp==3.19.2 +packaging==24.2 +more-itertools==10.3.0 +tomli==2.0.1 +jaraco.text==3.12.1 +inflect==7.3.1 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/files/wandb-metadata.json b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b19d17709c3576a60f291f698b22efb87217ffd4 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/files/wandb-metadata.json @@ -0,0 +1,148 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-23T17:22:09.267318Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_trained_raw_cleaned.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects", + "--cloud-save-path", + "/tmp/haozhezhao/MLLMG/checkpoint", + "--lr", + "5e-5", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_val_raw_cleaned.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/instructblip-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "200", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "56", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "4", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "2", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "instructblip", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--train_all", + "--load_fixed_llamagen", + "--save_total_limit", + "1", + "--fix", + "gpt-empty-fix", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/0117000.pt" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "git": { + "remote": "git@github.com:HaozheZhao/MLLMG.git", + "commit": "4b66c5ea70beaa476544d6b4715bf5055f7ab018" + }, + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects", + "host": "447cc403a8794092814259713c51c1df00001W", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "59238383616" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug-core.log b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..0a2c48d76c1583861b3479de24b280d566a30dd2 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-04-23T17:22:08.807661967Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp63dvsyhq/port-2476061.txt","pid":2476061,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-23T17:22:08.811287515Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2476061} +{"time":"2025-04-23T17:22:08.811564023Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":34413,"Zone":""}} +{"time":"2025-04-23T17:22:08.936431967Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:45338"} +{"time":"2025-04-23T17:22:09.27002438Z","level":"INFO","msg":"handleInformInit: received","streamId":"0k7f5yzn","id":"127.0.0.1:45338"} +{"time":"2025-04-23T17:22:09.479778004Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"0k7f5yzn","id":"127.0.0.1:45338"} +{"time":"2025-04-24T14:16:08.244176935Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug-internal.log b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d64c6d98e60a9634a7bbac7649c1dd8fdbdbc06c --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug-internal.log @@ -0,0 +1,9 @@ +{"time":"2025-04-23T17:22:09.270343768Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug-core.log"} +{"time":"2025-04-23T17:22:09.479730525Z","level":"INFO","msg":"created new stream","id":"0k7f5yzn"} +{"time":"2025-04-23T17:22:09.479771201Z","level":"INFO","msg":"stream: started","id":"0k7f5yzn"} +{"time":"2025-04-23T17:22:09.479797105Z","level":"INFO","msg":"writer: Do: started","stream_id":"0k7f5yzn"} +{"time":"2025-04-23T17:22:09.479818846Z","level":"INFO","msg":"sender: started","stream_id":"0k7f5yzn"} +{"time":"2025-04-23T17:22:09.479824946Z","level":"INFO","msg":"handler: started","stream_id":"0k7f5yzn"} +{"time":"2025-04-23T17:22:09.853827349Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-23T19:47:10.551735778Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/0k7f5yzn/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-04-23T19:50:13.944208334Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/0k7f5yzn/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug.log b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..07ac85abafd53edea355af0191fae1e6814b7998 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug.log @@ -0,0 +1,22 @@ +2025-04-23 17:22:09,261 INFO MainThread:2476061 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-23 17:22:09,261 INFO MainThread:2476061 [wandb_setup.py:_flush():67] Configure stats pid to 2476061 +2025-04-23 17:22:09,261 INFO MainThread:2476061 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-23 17:22:09,261 INFO MainThread:2476061 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-23 17:22:09,261 INFO MainThread:2476061 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-23 17:22:09,261 INFO MainThread:2476061 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug.log +2025-04-23 17:22:09,261 INFO MainThread:2476061 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/logs/debug-internal.log +2025-04-23 17:22:09,261 INFO MainThread:2476061 [wandb_init.py:init():761] calling init triggers +2025-04-23 17:22:09,261 INFO MainThread:2476061 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_trained_raw_cleaned.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/0117000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 5e-05, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_val_raw_cleaned.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/instructblip-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 200, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'instructblip', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': False, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': None, 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-23 17:22:09,261 INFO MainThread:2476061 [wandb_init.py:init():784] starting backend +2025-04-23 17:22:09,261 INFO MainThread:2476061 [wandb_init.py:init():788] sending inform_init request +2025-04-23 17:22:09,267 INFO MainThread:2476061 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-23 17:22:09,267 INFO MainThread:2476061 [wandb_init.py:init():798] backend started and connected +2025-04-23 17:22:09,269 INFO MainThread:2476061 [wandb_init.py:init():891] updated telemetry +2025-04-23 17:22:09,275 INFO MainThread:2476061 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-23 17:22:09,850 INFO MainThread:2476061 [wandb_init.py:init():990] starting run threads in backend +2025-04-23 17:22:09,958 INFO MainThread:2476061 [wandb_run.py:_console_start():2375] atexit reg +2025-04-23 17:22:09,958 INFO MainThread:2476061 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-23 17:22:09,959 INFO MainThread:2476061 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-23 17:22:09,959 INFO MainThread:2476061 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-23 17:22:09,961 INFO MainThread:2476061 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/run-0k7f5yzn.wandb b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/run-0k7f5yzn.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f33d07bb18793975eeb55a6e441757898b6932f5 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/run-0k7f5yzn.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db9f4e2ba7484116e3179380d287de6bf0545989bc4039c60978a7cb9d30713d +size 36831232 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/config.yaml b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d651cf1e6530dc02865cfdb2e70abffa4fd2809b --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/config.yaml @@ -0,0 +1,199 @@ +_wandb: + value: + cli_version: 0.19.8 + m: [] + python_version: 3.11.11 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "3": + - 13 + - 16 + - 23 + - 55 + - 61 + "4": 3.11.11 + "5": 0.19.8 + "6": 4.49.0 + "8": + - 5 + "12": 0.19.8 + "13": linux-x86_64 +beta1: + value: 0.9 +beta2: + value: 0.95 +cfg_scale: + value: 7.5 +ckpt_every: + value: 2000 +class_dropout_prob: + value: 0.1 +cloud_save_path: + value: /tmp/haozhezhao/MLLMG/checkpoint +cls_token_num: + value: 512 +codebook_embed_dim: + value: 8 +codebook_size: + value: 16384 +continue_stage1: + value: false +data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_trained_raw_cleaned.jsonl +dataset: + value: ti2i +dist_backend: + value: nccl +dist_url: + value: env:// +distributed: + value: true +do_eval: + value: true +do_recovery: + value: true +downsample_size: + value: 16 +dreambench_eval: + value: false +drop_path: + value: 0 +dropout_p: + value: 0.1 +epochs: + value: 2 +eval_steps: + value: 2000 +find_unused_parameters: + value: true +fix: + value: gpt-empty-fix +global_batch_size: + value: 56 +global_seed: + value: 0 +gpt_ckpt: + value: /tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0056000.pt +gpt_model: + value: GPT-XL +gpt_type: + value: t2i +gpu: + value: 0 +gradient_accumulation_steps: + value: 4 +image_only_rate: + value: 0.1 +image_place_holder: + value: +image_size: + value: 512 +load_fixed_llamagen: + value: true +load_from_checkpoint: + value: /tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt +load_language_projection: + value: null +load_subject_embedding: + value: null +load_visual_encoder: + value: false +log_every: + value: 25 +lr: + value: 5e-05 +lr_decay_ratio: + value: 0.1 +lr_decay_style: + value: cosine +max_eval_samples: + value: 200 +max_grad_norm: + value: 1 +mixed_precision: + value: bf16 +mm_vision_tower: + value: openai/clip-vit-large-patch14 +model_name_or_path: + value: /tmp/haozhezhao/model/instructblip-flan-t5-xl +multimodal_encoder: + value: instructblip +no_compile: + value: false +no_left_padding: + value: false +no_local_save: + value: false +no_replace: + value: false +num_classes: + value: 1000 +num_workers: + value: 4 +processor_path: + value: null +project_name: + value: llamagen_ti2i +rank: + value: 0 +reference_data_path: + value: /tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl +replace_subject: + value: false +results_dir: + value: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects +resume: + value: true +save_total_limit: + value: 1 +stage2: + value: false +subject_driven: + value: true +temperature: + value: 0.9 +token_dropout_p: + value: 0.1 +top_k: + value: 16384 +top_p: + value: 1 +train_all: + value: true +train_iters: + value: 500000 +train_text_encoder: + value: true +unfreeze_output: + value: false +use_vision_tower: + value: true +val_data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_val_raw_cleaned.jsonl +vocab_size: + value: 16384 +vq_ckpt: + value: /tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt +vq_model: + value: VQ-16 +warmup: + value: 0.05 +weight_decay: + value: 0.05 +with_image_only: + value: false +world_size: + value: 8 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/output.log b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..63b55cb7d7c64484c73fefd863fe82a109123a65 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/output.log @@ -0,0 +1,1716 @@ +[2025-04-24 14:31:24] Training for 2 epochs... +[2025-04-24 14:31:24] Beginning epoch 1... + 22%|███████████████████████████████▍ | 10081/45917 [12:33<44:11, 13.52it/s]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 22%|███████████████████████████████▍ | 10081/45917 [12:50<44:11, 13.52it/s][rank0]:W0424 14:45:28.059000 2575513 site-packages/torch/_dynamo/variables/tensor.py:776] [6/0] Graph break from `Tensor.item()`, consider setting: +[rank0]:W0424 14:45:28.059000 2575513 site-packages/torch/_dynamo/variables/tensor.py:776] [6/0] torch._dynamo.config.capture_scalar_outputs = True +[rank0]:W0424 14:45:28.059000 2575513 site-packages/torch/_dynamo/variables/tensor.py:776] [6/0] or: +[rank0]:W0424 14:45:28.059000 2575513 site-packages/torch/_dynamo/variables/tensor.py:776] [6/0] env TORCHDYNAMO_CAPTURE_SCALAR_OUTPUTS=1 +[rank0]:W0424 14:45:28.059000 2575513 site-packages/torch/_dynamo/variables/tensor.py:776] [6/0] to include these operations in the captured graph. +[rank0]:W0424 14:45:28.059000 2575513 site-packages/torch/_dynamo/variables/tensor.py:776] [6/0] +[rank0]:W0424 14:45:28.059000 2575513 site-packages/torch/_dynamo/variables/tensor.py:776] [6/0] Graph break: from user code at: +[rank0]:W0424 14:45:28.059000 2575513 site-packages/torch/_dynamo/variables/tensor.py:776] [6/0] File "/tmp/haozhezhao/MLLMG/autoregressive/models/empty_fix_gpt.py", line 483, in torch_dynamo_resume_in_forward_at_477 +[rank0]:W0424 14:45:28.059000 2575513 site-packages/torch/_dynamo/variables/tensor.py:776] [6/0] t5_feat_len = t5_attn.sum().item() +[rank0]:W0424 14:45:28.059000 2575513 site-packages/torch/_dynamo/variables/tensor.py:776] [6/0] +[rank0]:W0424 14:45:28.059000 2575513 site-packages/torch/_dynamo/variables/tensor.py:776] [6/0] + 22%|██████████████████████████████▌ | 10084/45917 [15:06<125:57:38, 12.65s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 22%|██████████████████████████████▌ | 10085/45917 [17:17<250:19:28, 25.15s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 22%|██████████████████████████████▌ | 10087/45917 [20:32<443:08:38, 44.52s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 22%|██████████████████████████████▌ | 10088/45917 [22:07<535:27:21, 53.80s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 22%|██████████████████████████████▌ | 10090/45917 [24:57<662:00:47, 66.52s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 22%|██████████████████████████████▌ | 10091/45917 [26:14<686:35:58, 68.99s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 22%|██████████████████████████████▌ | 10095/45917 [31:51<780:00:08, 78.39s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 22%|██████████████████████████████▌ | 10098/45917 [36:11<819:27:58, 82.36s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 22%|██████████████████████████████▌ | 10107/45917 [43:49<636:12:41, 63.96s/it][2025-04-24 15:15:15] (step=0056025) Train Loss: 6.2975, Train Steps/Sec: 0.01 + 22%|██████████████████████████████▊ | 10115/45917 [45:05<70:35:08, 7.10s/it][rank0]:W0424 15:16:30.439000 2575513 site-packages/torch/_dynamo/convert_frame.py:844] [4/8] torch._dynamo hit config.cache_size_limit (8) +[rank0]:W0424 15:16:30.439000 2575513 site-packages/torch/_dynamo/convert_frame.py:844] [4/8] function: 'torch_dynamo_resume_in_forward_at_1440' (/tmp/haozhezhao/MLLMG/autoregressive/models/instructblip/modeling_instructblip.py:1440) +[rank0]:W0424 15:16:30.439000 2575513 site-packages/torch/_dynamo/convert_frame.py:844] [4/8] last reason: 4/0: tensor 'L['___stack0']' size mismatch at index 0. expected 15, actual 13 +[rank0]:W0424 15:16:30.439000 2575513 site-packages/torch/_dynamo/convert_frame.py:844] [4/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank0]:W0424 15:16:30.439000 2575513 site-packages/torch/_dynamo/convert_frame.py:844] [4/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html. + 22%|██████████████████████████████▊ | 10116/45917 [45:06<52:39:06, 5.29s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 22%|██████████████████████████████▋ | 10132/45917 [48:16<135:38:40, 13.65s/it][2025-04-24 15:19:42] (step=0056050) Train Loss: 6.3270, Train Steps/Sec: 0.09 + 22%|██████████████████████████████▉ | 10157/45917 [48:52<13:43:07, 1.38s/it][2025-04-24 15:20:17] (step=0056075) Train Loss: 6.3124, Train Steps/Sec: 0.71 + 22%|███████████████████████████████ | 10182/45917 [49:19<10:54:51, 1.10s/it][2025-04-24 15:20:45] (step=0056100) Train Loss: 6.2960, Train Steps/Sec: 0.90 + 22%|███████████████████████████████ | 10207/45917 [49:47<11:05:24, 1.12s/it][2025-04-24 15:21:13] (step=0056125) Train Loss: 6.3331, Train Steps/Sec: 0.91 + 22%|███████████████████████████████▏ | 10232/45917 [50:14<10:59:57, 1.11s/it][2025-04-24 15:21:40] (step=0056150) Train Loss: 6.3175, Train Steps/Sec: 0.91 + 22%|███████████████████████████████▎ | 10257/45917 [50:42<10:55:33, 1.10s/it][2025-04-24 15:22:08] (step=0056175) Train Loss: 6.3137, Train Steps/Sec: 0.90 + 22%|███████████████████████████████▎ | 10282/45917 [51:10<10:59:58, 1.11s/it][2025-04-24 15:22:36] (step=0056200) Train Loss: 6.3111, Train Steps/Sec: 0.90 + 22%|███████████████████████████████▍ | 10307/45917 [51:45<11:57:51, 1.21s/it][2025-04-24 15:23:11] (step=0056225) Train Loss: 6.3284, Train Steps/Sec: 0.71 + 23%|███████████████████████████████▌ | 10332/45917 [52:13<10:56:07, 1.11s/it][2025-04-24 15:23:39] (step=0056250) Train Loss: 6.3305, Train Steps/Sec: 0.90 + 23%|███████████████████████████████▌ | 10357/45917 [52:41<10:49:23, 1.10s/it][2025-04-24 15:24:06] (step=0056275) Train Loss: 6.3283, Train Steps/Sec: 0.91 + 23%|███████████████████████████████▋ | 10382/45917 [53:08<10:50:45, 1.10s/it][2025-04-24 15:24:34] (step=0056300) Train Loss: 6.3770, Train Steps/Sec: 0.90 + 23%|███████████████████████████████▋ | 10407/45917 [53:36<11:07:20, 1.13s/it][2025-04-24 15:25:02] (step=0056325) Train Loss: 6.3261, Train Steps/Sec: 0.90 + 23%|███████████████████████████████▊ | 10432/45917 [54:04<10:52:27, 1.10s/it][2025-04-24 15:25:29] (step=0056350) Train Loss: 6.3704, Train Steps/Sec: 0.91 + 23%|███████████████████████████████▉ | 10457/45917 [54:31<10:50:04, 1.10s/it][2025-04-24 15:25:57] (step=0056375) Train Loss: 6.3315, Train Steps/Sec: 0.90 + 23%|███████████████████████████████▉ | 10482/45917 [54:59<10:45:04, 1.09s/it][2025-04-24 15:26:25] (step=0056400) Train Loss: 6.3438, Train Steps/Sec: 0.90 + 23%|████████████████████████████████ | 10507/45917 [55:27<11:02:32, 1.12s/it][2025-04-24 15:26:53] (step=0056425) Train Loss: 6.3129, Train Steps/Sec: 0.90 + 23%|████████████████████████████████ | 10532/45917 [55:55<10:55:28, 1.11s/it][2025-04-24 15:27:20] (step=0056450) Train Loss: 6.3091, Train Steps/Sec: 0.90 + 23%|████████████████████████████████▏ | 10557/45917 [56:22<10:40:31, 1.09s/it][2025-04-24 15:27:48] (step=0056475) Train Loss: 6.3124, Train Steps/Sec: 0.91 + 23%|████████████████████████████████▎ | 10582/45917 [56:50<10:47:34, 1.10s/it][2025-04-24 15:28:16] (step=0056500) Train Loss: 6.3059, Train Steps/Sec: 0.90 + 23%|████████████████████████████████▎ | 10607/45917 [57:18<11:01:31, 1.12s/it][2025-04-24 15:28:43] (step=0056525) Train Loss: 6.3558, Train Steps/Sec: 0.91 + 23%|████████████████████████████████▍ | 10632/45917 [57:45<10:55:45, 1.12s/it][2025-04-24 15:29:11] (step=0056550) Train Loss: 6.3492, Train Steps/Sec: 0.90 + 23%|████████████████████████████████▍ | 10657/45917 [58:13<10:47:26, 1.10s/it][2025-04-24 15:29:39] (step=0056575) Train Loss: 6.3300, Train Steps/Sec: 0.91 + 23%|████████████████████████████████▌ | 10682/45917 [58:41<10:37:04, 1.08s/it][2025-04-24 15:30:07] (step=0056600) Train Loss: 6.2586, Train Steps/Sec: 0.90 + 23%|████████████████████████████████▋ | 10707/45917 [59:08<10:59:43, 1.12s/it][2025-04-24 15:30:34] (step=0056625) Train Loss: 6.3573, Train Steps/Sec: 0.90 + 23%|████████████████████████████████▋ | 10732/45917 [59:36<10:46:26, 1.10s/it][2025-04-24 15:31:02] (step=0056650) Train Loss: 6.2588, Train Steps/Sec: 0.90 + 23%|████████████████████████████████▎ | 10757/45917 [1:00:04<10:45:09, 1.10s/it][2025-04-24 15:31:30] (step=0056675) Train Loss: 6.2753, Train Steps/Sec: 0.90 + 23%|████████████████████████████████▍ | 10782/45917 [1:00:32<10:42:11, 1.10s/it][2025-04-24 15:31:57] (step=0056700) Train Loss: 6.2903, Train Steps/Sec: 0.90 + 24%|████████████████████████████████▍ | 10807/45917 [1:00:59<10:56:34, 1.12s/it][2025-04-24 15:32:25] (step=0056725) Train Loss: 6.3366, Train Steps/Sec: 0.91 + 24%|████████████████████████████████▌ | 10832/45917 [1:01:27<10:47:54, 1.11s/it][2025-04-24 15:32:53] (step=0056750) Train Loss: 6.2722, Train Steps/Sec: 0.90 + 24%|████████████████████████████████▋ | 10857/45917 [1:01:55<10:39:46, 1.09s/it][2025-04-24 15:33:21] (step=0056775) Train Loss: 6.3674, Train Steps/Sec: 0.91 + 24%|████████████████████████████████▋ | 10882/45917 [1:02:22<10:35:49, 1.09s/it][2025-04-24 15:33:48] (step=0056800) Train Loss: 6.3090, Train Steps/Sec: 0.90 + 24%|████████████████████████████████▊ | 10907/45917 [1:02:50<11:05:48, 1.14s/it][2025-04-24 15:34:16] (step=0056825) Train Loss: 6.2719, Train Steps/Sec: 0.90 + 24%|████████████████████████████████▊ | 10932/45917 [1:03:18<10:48:14, 1.11s/it][2025-04-24 15:34:44] (step=0056850) Train Loss: 6.3305, Train Steps/Sec: 0.90 + 24%|████████████████████████████████▉ | 10957/45917 [1:03:46<10:36:47, 1.09s/it][2025-04-24 15:35:12] (step=0056875) Train Loss: 6.3494, Train Steps/Sec: 0.90 + 24%|█████████████████████████████████ | 10982/45917 [1:04:14<10:35:27, 1.09s/it][2025-04-24 15:35:39] (step=0056900) Train Loss: 6.3308, Train Steps/Sec: 0.90 + 24%|█████████████████████████████████ | 11007/45917 [1:04:41<10:50:25, 1.12s/it][2025-04-24 15:36:07] (step=0056925) Train Loss: 6.3525, Train Steps/Sec: 0.90 + 24%|█████████████████████████████████▏ | 11032/45917 [1:05:09<10:42:42, 1.11s/it][2025-04-24 15:36:35] (step=0056950) Train Loss: 6.3302, Train Steps/Sec: 0.90 + 24%|█████████████████████████████████▏ | 11057/45917 [1:05:37<10:37:44, 1.10s/it][2025-04-24 15:37:03] (step=0056975) Train Loss: 6.3063, Train Steps/Sec: 0.90 + 24%|█████████████████████████████████▎ | 11082/45917 [1:06:04<10:40:14, 1.10s/it][2025-04-24 15:37:30] (step=0057000) Train Loss: 6.2988, Train Steps/Sec: 0.90 + 24%|█████████████████████████████████▍ | 11107/45917 [1:06:32<10:47:40, 1.12s/it][2025-04-24 15:37:58] (step=0057025) Train Loss: 6.3264, Train Steps/Sec: 0.90 + 24%|█████████████████████████████████▍ | 11132/45917 [1:07:00<10:40:46, 1.11s/it][2025-04-24 15:38:26] (step=0057050) Train Loss: 6.3505, Train Steps/Sec: 0.91 + 24%|█████████████████████████████████▌ | 11157/45917 [1:07:28<10:47:29, 1.12s/it][2025-04-24 15:38:53] (step=0057075) Train Loss: 6.3634, Train Steps/Sec: 0.90 + 24%|█████████████████████████████████▌ | 11182/45917 [1:07:55<10:36:02, 1.10s/it][2025-04-24 15:39:21] (step=0057100) Train Loss: 6.3218, Train Steps/Sec: 0.90 + 24%|█████████████████████████████████▋ | 11207/45917 [1:08:23<10:47:11, 1.12s/it][2025-04-24 15:39:49] (step=0057125) Train Loss: 6.3056, Train Steps/Sec: 0.91 + 24%|█████████████████████████████████▊ | 11232/45917 [1:08:51<10:40:59, 1.11s/it][2025-04-24 15:40:16] (step=0057150) Train Loss: 6.2911, Train Steps/Sec: 0.90 + 25%|█████████████████████████████████▊ | 11257/45917 [1:09:18<10:44:51, 1.12s/it][2025-04-24 15:40:44] (step=0057175) Train Loss: 6.2883, Train Steps/Sec: 0.90 + 25%|█████████████████████████████████▉ | 11282/45917 [1:09:46<10:35:15, 1.10s/it][2025-04-24 15:41:12] (step=0057200) Train Loss: 6.3485, Train Steps/Sec: 0.90 + 25%|█████████████████████████████████▉ | 11307/45917 [1:10:14<10:48:54, 1.12s/it][2025-04-24 15:41:40] (step=0057225) Train Loss: 6.3157, Train Steps/Sec: 0.90 + 25%|██████████████████████████████████ | 11332/45917 [1:10:42<10:38:15, 1.11s/it][2025-04-24 15:42:08] (step=0057250) Train Loss: 6.3210, Train Steps/Sec: 0.90 + 25%|██████████████████████████████████▏ | 11357/45917 [1:11:10<10:32:09, 1.10s/it][2025-04-24 15:42:35] (step=0057275) Train Loss: 6.2788, Train Steps/Sec: 0.90 + 25%|██████████████████████████████████▏ | 11382/45917 [1:11:38<10:41:29, 1.11s/it][2025-04-24 15:43:03] (step=0057300) Train Loss: 6.2808, Train Steps/Sec: 0.89 + 25%|██████████████████████████████████▎ | 11407/45917 [1:12:05<10:47:41, 1.13s/it][2025-04-24 15:43:31] (step=0057325) Train Loss: 6.2922, Train Steps/Sec: 0.90 + 25%|██████████████████████████████████▎ | 11432/45917 [1:12:33<10:37:13, 1.11s/it][2025-04-24 15:43:59] (step=0057350) Train Loss: 6.3324, Train Steps/Sec: 0.90 + 25%|██████████████████████████████████▍ | 11457/45917 [1:13:01<10:30:39, 1.10s/it][2025-04-24 15:44:27] (step=0057375) Train Loss: 6.2973, Train Steps/Sec: 0.90 + 25%|██████████████████████████████████▌ | 11482/45917 [1:13:28<10:31:00, 1.10s/it][2025-04-24 15:44:54] (step=0057400) Train Loss: 6.3573, Train Steps/Sec: 0.90 + 25%|██████████████████████████████████▌ | 11507/45917 [1:13:56<10:47:58, 1.13s/it][2025-04-24 15:45:22] (step=0057425) Train Loss: 6.2836, Train Steps/Sec: 0.91 + 25%|██████████████████████████████████▋ | 11532/45917 [1:14:24<10:31:02, 1.10s/it][2025-04-24 15:45:49] (step=0057450) Train Loss: 6.3219, Train Steps/Sec: 0.91 + 25%|██████████████████████████████████▋ | 11557/45917 [1:14:51<10:30:05, 1.10s/it][2025-04-24 15:46:17] (step=0057475) Train Loss: 6.3027, Train Steps/Sec: 0.90 + 25%|██████████████████████████████████▊ | 11582/45917 [1:15:19<10:29:44, 1.10s/it][2025-04-24 15:46:45] (step=0057500) Train Loss: 6.3505, Train Steps/Sec: 0.90 + 25%|██████████████████████████████████▉ | 11607/45917 [1:15:47<10:40:45, 1.12s/it][2025-04-24 15:47:13] (step=0057525) Train Loss: 6.2579, Train Steps/Sec: 0.91 + 25%|██████████████████████████████████▉ | 11632/45917 [1:16:15<10:36:03, 1.11s/it][2025-04-24 15:47:40] (step=0057550) Train Loss: 6.3601, Train Steps/Sec: 0.90 + 25%|███████████████████████████████████ | 11657/45917 [1:16:42<10:31:17, 1.11s/it][2025-04-24 15:48:08] (step=0057575) Train Loss: 6.3298, Train Steps/Sec: 0.91 + 25%|███████████████████████████████████ | 11682/45917 [1:17:10<10:32:41, 1.11s/it][2025-04-24 15:48:36] (step=0057600) Train Loss: 6.3833, Train Steps/Sec: 0.90 + 25%|███████████████████████████████████▏ | 11707/45917 [1:17:38<10:39:42, 1.12s/it][2025-04-24 15:49:03] (step=0057625) Train Loss: 6.3479, Train Steps/Sec: 0.90 + 26%|███████████████████████████████████▎ | 11732/45917 [1:18:05<10:31:10, 1.11s/it][2025-04-24 15:49:31] (step=0057650) Train Loss: 6.3426, Train Steps/Sec: 0.90 + 26%|███████████████████████████████████▎ | 11757/45917 [1:18:33<10:35:16, 1.12s/it][2025-04-24 15:49:59] (step=0057675) Train Loss: 6.3262, Train Steps/Sec: 0.90 + 26%|███████████████████████████████████▍ | 11782/45917 [1:19:01<10:22:15, 1.09s/it][2025-04-24 15:50:27] (step=0057700) Train Loss: 6.3230, Train Steps/Sec: 0.90 + 26%|███████████████████████████████████▍ | 11807/45917 [1:19:29<10:42:10, 1.13s/it][2025-04-24 15:50:54] (step=0057725) Train Loss: 6.3434, Train Steps/Sec: 0.90 + 26%|███████████████████████████████████▌ | 11832/45917 [1:19:56<10:27:17, 1.10s/it][2025-04-24 15:51:22] (step=0057750) Train Loss: 6.3370, Train Steps/Sec: 0.90 + 26%|███████████████████████████████████▋ | 11857/45917 [1:20:24<10:27:22, 1.11s/it][2025-04-24 15:51:50] (step=0057775) Train Loss: 6.2952, Train Steps/Sec: 0.90 + 26%|███████████████████████████████████▋ | 11882/45917 [1:20:52<10:19:17, 1.09s/it][2025-04-24 15:52:18] (step=0057800) Train Loss: 6.3414, Train Steps/Sec: 0.90 + 26%|███████████████████████████████████▊ | 11907/45917 [1:21:20<10:45:57, 1.14s/it][2025-04-24 15:52:45] (step=0057825) Train Loss: 6.3148, Train Steps/Sec: 0.90 + 26%|███████████████████████████████████▊ | 11932/45917 [1:21:47<10:22:12, 1.10s/it][2025-04-24 15:53:13] (step=0057850) Train Loss: 6.3360, Train Steps/Sec: 0.90 + 26%|███████████████████████████████████▉ | 11957/45917 [1:22:15<10:19:22, 1.09s/it][2025-04-24 15:53:41] (step=0057875) Train Loss: 6.2990, Train Steps/Sec: 0.91 + 26%|████████████████████████████████████ | 11982/45917 [1:22:43<10:19:05, 1.09s/it][2025-04-24 15:54:08] (step=0057900) Train Loss: 6.3239, Train Steps/Sec: 0.90 + 26%|████████████████████████████████████ | 12007/45917 [1:23:10<10:29:40, 1.11s/it][2025-04-24 15:54:36] (step=0057925) Train Loss: 6.3626, Train Steps/Sec: 0.91 + 26%|████████████████████████████████████▏ | 12032/45917 [1:23:38<10:24:26, 1.11s/it][2025-04-24 15:55:04] (step=0057950) Train Loss: 6.2671, Train Steps/Sec: 0.91 + 26%|████████████████████████████████████▏ | 12057/45917 [1:24:05<10:21:19, 1.10s/it][2025-04-24 15:55:31] (step=0057975) Train Loss: 6.2929, Train Steps/Sec: 0.91 + 26%|████████████████████████████████████▎ | 12082/45917 [1:24:33<10:16:40, 1.09s/it][2025-04-24 15:55:59] (step=0058000) Train Loss: 6.3211, Train Steps/Sec: 0.90 +[2025-04-24 15:55:59] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 15:55:59] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 15:55:59] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/3 [00:00 +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 59.05s/it] +[2025-04-24 16:43:00] Finish Eval in 60000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.64s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 16:43:21] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0060000.pt +[2025-04-24 16:43:23] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0058000.pt + 31%|██████████████████████████████████████████▍ | 14107/45917 [2:12:26<10:07:48, 1.15s/it][2025-04-24 16:43:52] (step=0060025) Train Loss: 6.3503, Train Steps/Sec: 0.07 + 31%|██████████████████████████████████████████▊ | 14132/45917 [2:12:53<9:48:52, 1.11s/it][2025-04-24 16:44:19] (step=0060050) Train Loss: 6.3163, Train Steps/Sec: 0.90 + 31%|██████████████████████████████████████████▊ | 14157/45917 [2:13:21<9:43:03, 1.10s/it][2025-04-24 16:44:47] (step=0060075) Train Loss: 6.3044, Train Steps/Sec: 0.90 + 31%|██████████████████████████████████████████▉ | 14182/45917 [2:13:49<9:38:36, 1.09s/it][2025-04-24 16:45:15] (step=0060100) Train Loss: 6.3417, Train Steps/Sec: 0.90 + 31%|███████████████████████████████████████████ | 14207/45917 [2:14:17<9:49:47, 1.12s/it][2025-04-24 16:45:43] (step=0060125) Train Loss: 6.3221, Train Steps/Sec: 0.90 + 31%|███████████████████████████████████████████ | 14232/45917 [2:14:45<9:42:24, 1.10s/it][2025-04-24 16:46:10] (step=0060150) Train Loss: 6.3354, Train Steps/Sec: 0.90 + 31%|██████████████████████████████████████████▊ | 14257/45917 [2:15:19<15:55:45, 1.81s/it][2025-04-24 16:46:45] (step=0060175) Train Loss: 6.2917, Train Steps/Sec: 0.72 + 31%|███████████████████████████████████████████▏ | 14282/45917 [2:15:47<9:36:01, 1.09s/it][2025-04-24 16:47:13] (step=0060200) Train Loss: 6.3356, Train Steps/Sec: 0.90 + 31%|███████████████████████████████████████████▎ | 14307/45917 [2:16:15<9:53:58, 1.13s/it][2025-04-24 16:47:41] (step=0060225) Train Loss: 6.2847, Train Steps/Sec: 0.90 + 31%|███████████████████████████████████████████▍ | 14332/45917 [2:16:43<9:46:41, 1.11s/it][2025-04-24 16:48:08] (step=0060250) Train Loss: 6.3315, Train Steps/Sec: 0.90 + 31%|███████████████████████████████████████████▍ | 14357/45917 [2:17:10<9:37:17, 1.10s/it][2025-04-24 16:48:36] (step=0060275) Train Loss: 6.3455, Train Steps/Sec: 0.90 + 31%|███████████████████████████████████████████▌ | 14382/45917 [2:17:38<9:32:54, 1.09s/it][2025-04-24 16:49:04] (step=0060300) Train Loss: 6.3324, Train Steps/Sec: 0.90 + 31%|███████████████████████████████████████████▌ | 14407/45917 [2:18:06<9:51:18, 1.13s/it][2025-04-24 16:49:32] (step=0060325) Train Loss: 6.3229, Train Steps/Sec: 0.90 + 31%|███████████████████████████████████████████▋ | 14432/45917 [2:18:33<9:37:05, 1.10s/it][2025-04-24 16:49:59] (step=0060350) Train Loss: 6.3650, Train Steps/Sec: 0.91 + 31%|███████████████████████████████████████████▊ | 14457/45917 [2:19:01<9:34:41, 1.10s/it][2025-04-24 16:50:27] (step=0060375) Train Loss: 6.3150, Train Steps/Sec: 0.90 + 32%|███████████████████████████████████████████▊ | 14482/45917 [2:19:29<9:37:05, 1.10s/it][2025-04-24 16:50:55] (step=0060400) Train Loss: 6.2967, Train Steps/Sec: 0.90 + 32%|███████████████████████████████████████████▉ | 14507/45917 [2:19:57<9:47:02, 1.12s/it][2025-04-24 16:51:22] (step=0060425) Train Loss: 6.3192, Train Steps/Sec: 0.90 + 32%|███████████████████████████████████████████▉ | 14532/45917 [2:20:24<9:36:14, 1.10s/it][2025-04-24 16:51:50] (step=0060450) Train Loss: 6.2796, Train Steps/Sec: 0.91 + 32%|████████████████████████████████████████████ | 14557/45917 [2:20:52<9:31:46, 1.09s/it][2025-04-24 16:52:18] (step=0060475) Train Loss: 6.3173, Train Steps/Sec: 0.90 + 32%|████████████████████████████████████████████▏ | 14582/45917 [2:21:20<9:33:05, 1.10s/it][2025-04-24 16:52:46] (step=0060500) Train Loss: 6.3102, Train Steps/Sec: 0.90 + 32%|████████████████████████████████████████████▏ | 14607/45917 [2:21:48<9:48:12, 1.13s/it][2025-04-24 16:53:13] (step=0060525) Train Loss: 6.3404, Train Steps/Sec: 0.90 + 32%|████████████████████████████████████████████▎ | 14632/45917 [2:22:15<9:37:22, 1.11s/it][2025-04-24 16:53:41] (step=0060550) Train Loss: 6.3216, Train Steps/Sec: 0.90 + 32%|████████████████████████████████████████████▎ | 14657/45917 [2:22:50<9:30:17, 1.09s/it][2025-04-24 16:54:15] (step=0060575) Train Loss: 6.3043, Train Steps/Sec: 0.73 + 32%|████████████████████████████████████████████▍ | 14682/45917 [2:23:17<9:27:15, 1.09s/it][2025-04-24 16:54:43] (step=0060600) Train Loss: 6.3224, Train Steps/Sec: 0.90 + 32%|████████████████████████████████████████████▌ | 14707/45917 [2:23:45<9:48:44, 1.13s/it][2025-04-24 16:55:11] (step=0060625) Train Loss: 6.3313, Train Steps/Sec: 0.90 + 32%|████████████████████████████████████████████▌ | 14732/45917 [2:24:13<9:33:31, 1.10s/it][2025-04-24 16:55:38] (step=0060650) Train Loss: 6.2895, Train Steps/Sec: 0.91 + 32%|████████████████████████████████████████████▋ | 14757/45917 [2:24:40<9:31:55, 1.10s/it][2025-04-24 16:56:06] (step=0060675) Train Loss: 6.3532, Train Steps/Sec: 0.91 + 32%|████████████████████████████████████████████▍ | 14782/45917 [2:25:15<22:19:30, 2.58s/it][2025-04-24 16:56:41] (step=0060700) Train Loss: 6.3282, Train Steps/Sec: 0.72 + 32%|████████████████████████████████████████████▊ | 14807/45917 [2:25:43<9:46:16, 1.13s/it][2025-04-24 16:57:09] (step=0060725) Train Loss: 6.3371, Train Steps/Sec: 0.90 + 32%|████████████████████████████████████████████▌ | 14832/45917 [2:26:17<12:21:23, 1.43s/it][2025-04-24 16:57:43] (step=0060750) Train Loss: 6.3280, Train Steps/Sec: 0.73 + 32%|████████████████████████████████████████████▉ | 14857/45917 [2:26:45<9:29:18, 1.10s/it][2025-04-24 16:58:11] (step=0060775) Train Loss: 6.3117, Train Steps/Sec: 0.90 + 32%|█████████████████████████████████████████████ | 14882/45917 [2:27:13<9:24:12, 1.09s/it][2025-04-24 16:58:38] (step=0060800) Train Loss: 6.2928, Train Steps/Sec: 0.90 + 32%|█████████████████████████████████████████████▏ | 14907/45917 [2:27:40<9:40:56, 1.12s/it][2025-04-24 16:59:06] (step=0060825) Train Loss: 6.2850, Train Steps/Sec: 0.90 + 33%|█████████████████████████████████████████████▏ | 14932/45917 [2:28:08<9:28:08, 1.10s/it][2025-04-24 16:59:34] (step=0060850) Train Loss: 6.3255, Train Steps/Sec: 0.91 + 33%|█████████████████████████████████████████████▎ | 14957/45917 [2:28:42<9:32:32, 1.11s/it][2025-04-24 17:00:08] (step=0060875) Train Loss: 6.2885, Train Steps/Sec: 0.73 + 33%|█████████████████████████████████████████████▎ | 14982/45917 [2:29:10<9:25:43, 1.10s/it][2025-04-24 17:00:36] (step=0060900) Train Loss: 6.3442, Train Steps/Sec: 0.89 + 33%|█████████████████████████████████████████████ | 15007/45917 [2:29:43<10:00:42, 1.17s/it][2025-04-24 17:01:09] (step=0060925) Train Loss: 6.3028, Train Steps/Sec: 0.76 + 33%|█████████████████████████████████████████████▌ | 15032/45917 [2:30:11<9:35:10, 1.12s/it][2025-04-24 17:01:37] (step=0060950) Train Loss: 6.3116, Train Steps/Sec: 0.90 + 33%|█████████████████████████████████████████████▌ | 15057/45917 [2:30:39<9:24:25, 1.10s/it][2025-04-24 17:02:05] (step=0060975) Train Loss: 6.2935, Train Steps/Sec: 0.90 + 33%|█████████████████████████████████████████████▋ | 15082/45917 [2:31:07<9:26:27, 1.10s/it][2025-04-24 17:02:32] (step=0061000) Train Loss: 6.3121, Train Steps/Sec: 0.90 + 33%|█████████████████████████████████████████████▋ | 15107/45917 [2:31:35<9:43:02, 1.14s/it][2025-04-24 17:03:00] (step=0061025) Train Loss: 6.3369, Train Steps/Sec: 0.90 + 33%|█████████████████████████████████████████████▊ | 15132/45917 [2:32:08<9:44:26, 1.14s/it][2025-04-24 17:03:34] (step=0061050) Train Loss: 6.3160, Train Steps/Sec: 0.75 + 33%|█████████████████████████████████████████████▉ | 15157/45917 [2:32:36<9:30:49, 1.11s/it][2025-04-24 17:04:01] (step=0061075) Train Loss: 6.3162, Train Steps/Sec: 0.90 + 33%|█████████████████████████████████████████████▉ | 15182/45917 [2:33:03<9:27:26, 1.11s/it][2025-04-24 17:04:29] (step=0061100) Train Loss: 6.2995, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████████████████ | 15207/45917 [2:33:31<9:36:08, 1.13s/it][2025-04-24 17:04:57] (step=0061125) Train Loss: 6.2939, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████████████████ | 15232/45917 [2:33:59<9:25:06, 1.10s/it][2025-04-24 17:05:25] (step=0061150) Train Loss: 6.3304, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████████████████▏ | 15257/45917 [2:34:27<9:27:08, 1.11s/it][2025-04-24 17:05:53] (step=0061175) Train Loss: 6.3099, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████████████████▎ | 15282/45917 [2:34:55<9:23:21, 1.10s/it][2025-04-24 17:06:21] (step=0061200) Train Loss: 6.3784, Train Steps/Sec: 0.89 + 33%|██████████████████████████████████████████████▎ | 15307/45917 [2:35:23<9:38:10, 1.13s/it][2025-04-24 17:06:49] (step=0061225) Train Loss: 6.3749, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████████████████▍ | 15332/45917 [2:35:51<9:28:41, 1.12s/it][2025-04-24 17:07:16] (step=0061250) Train Loss: 6.3284, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████████████████▍ | 15357/45917 [2:36:19<9:23:46, 1.11s/it][2025-04-24 17:07:44] (step=0061275) Train Loss: 6.3310, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████████████████▌ | 15382/45917 [2:36:46<9:21:34, 1.10s/it][2025-04-24 17:08:12] (step=0061300) Train Loss: 6.3179, Train Steps/Sec: 0.90 + 34%|██████████████████████████████████████████████▋ | 15407/45917 [2:37:14<9:34:16, 1.13s/it][2025-04-24 17:08:40] (step=0061325) Train Loss: 6.3433, Train Steps/Sec: 0.90 + 34%|██████████████████████████████████████████████▋ | 15432/45917 [2:37:42<9:27:13, 1.12s/it][2025-04-24 17:09:08] (step=0061350) Train Loss: 6.3213, Train Steps/Sec: 0.90 + 34%|██████████████████████████████████████████████▊ | 15457/45917 [2:38:10<9:19:48, 1.10s/it][2025-04-24 17:09:35] (step=0061375) Train Loss: 6.3531, Train Steps/Sec: 0.90 + 34%|██████████████████████████████████████████████▊ | 15482/45917 [2:38:37<9:21:34, 1.11s/it][2025-04-24 17:10:03] (step=0061400) Train Loss: 6.3109, Train Steps/Sec: 0.90 + 34%|██████████████████████████████████████████████▉ | 15507/45917 [2:39:05<9:28:05, 1.12s/it][2025-04-24 17:10:31] (step=0061425) Train Loss: 6.3233, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████████████████ | 15532/45917 [2:39:33<9:24:41, 1.12s/it][2025-04-24 17:10:59] (step=0061450) Train Loss: 6.3188, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████████████████ | 15557/45917 [2:40:01<9:21:03, 1.11s/it][2025-04-24 17:11:27] (step=0061475) Train Loss: 6.3384, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████████████████▏ | 15582/45917 [2:40:29<9:21:01, 1.11s/it][2025-04-24 17:11:55] (step=0061500) Train Loss: 6.3512, Train Steps/Sec: 0.89 + 34%|███████████████████████████████████████████████▏ | 15607/45917 [2:40:57<9:34:09, 1.14s/it][2025-04-24 17:12:23] (step=0061525) Train Loss: 6.3021, Train Steps/Sec: 0.89 + 34%|███████████████████████████████████████████████▎ | 15632/45917 [2:41:25<9:17:17, 1.10s/it][2025-04-24 17:12:50] (step=0061550) Train Loss: 6.2827, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████████████████▍ | 15657/45917 [2:41:52<9:18:24, 1.11s/it][2025-04-24 17:13:18] (step=0061575) Train Loss: 6.2964, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████████████████▍ | 15682/45917 [2:42:20<9:16:18, 1.10s/it][2025-04-24 17:13:46] (step=0061600) Train Loss: 6.3260, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████████████████▌ | 15707/45917 [2:42:48<9:27:37, 1.13s/it][2025-04-24 17:14:14] (step=0061625) Train Loss: 6.2853, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████████████████▌ | 15732/45917 [2:43:16<9:14:23, 1.10s/it][2025-04-24 17:14:42] (step=0061650) Train Loss: 6.3327, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████████████████▋ | 15757/45917 [2:43:44<9:19:01, 1.11s/it][2025-04-24 17:15:10] (step=0061675) Train Loss: 6.3412, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████████████████▊ | 15782/45917 [2:44:12<9:16:12, 1.11s/it][2025-04-24 17:15:38] (step=0061700) Train Loss: 6.3378, Train Steps/Sec: 0.89 + 34%|███████████████████████████████████████████████▊ | 15807/45917 [2:44:40<9:27:53, 1.13s/it][2025-04-24 17:16:05] (step=0061725) Train Loss: 6.3189, Train Steps/Sec: 0.90 + 34%|███████████████████████████████████████████████▉ | 15832/45917 [2:45:07<9:17:21, 1.11s/it][2025-04-24 17:16:33] (step=0061750) Train Loss: 6.2872, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████ | 15857/45917 [2:45:35<9:17:47, 1.11s/it][2025-04-24 17:17:01] (step=0061775) Train Loss: 6.2915, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████ | 15882/45917 [2:46:03<9:13:40, 1.11s/it][2025-04-24 17:17:29] (step=0061800) Train Loss: 6.3449, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████▏ | 15907/45917 [2:46:31<9:25:51, 1.13s/it][2025-04-24 17:17:57] (step=0061825) Train Loss: 6.3186, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████▏ | 15932/45917 [2:46:59<9:18:30, 1.12s/it][2025-04-24 17:18:25] (step=0061850) Train Loss: 6.3165, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████▎ | 15957/45917 [2:47:27<9:09:07, 1.10s/it][2025-04-24 17:18:52] (step=0061875) Train Loss: 6.2633, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████▍ | 15982/45917 [2:47:54<9:06:44, 1.10s/it][2025-04-24 17:19:20] (step=0061900) Train Loss: 6.3440, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████▍ | 16007/45917 [2:48:22<9:24:19, 1.13s/it][2025-04-24 17:19:48] (step=0061925) Train Loss: 6.2830, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████▌ | 16032/45917 [2:48:50<9:08:07, 1.10s/it][2025-04-24 17:20:16] (step=0061950) Train Loss: 6.2966, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████▌ | 16057/45917 [2:49:18<9:09:41, 1.10s/it][2025-04-24 17:20:44] (step=0061975) Train Loss: 6.2768, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████▋ | 16082/45917 [2:49:46<9:12:53, 1.11s/it][2025-04-24 17:21:12] (step=0062000) Train Loss: 6.3538, Train Steps/Sec: 0.89 +[2025-04-24 17:21:12] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 17:21:12] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 17:21:12] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:59<00:00, 59.74s/it] +[2025-04-24 17:25:59] Finish Eval in 62000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.32s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 17:26:21] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0062000.pt +[2025-04-24 17:26:23] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0060000.pt + 35%|████████████████████████████████████████████████▊ | 16107/45917 [2:55:25<9:44:29, 1.18s/it][2025-04-24 17:26:51] (step=0062025) Train Loss: 6.3304, Train Steps/Sec: 0.07 + 35%|████████████████████████████████████████████████▊ | 16132/45917 [2:55:53<9:17:13, 1.12s/it][2025-04-24 17:27:19] (step=0062050) Train Loss: 6.3144, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████▉ | 16157/45917 [2:56:21<9:10:08, 1.11s/it][2025-04-24 17:27:47] (step=0062075) Train Loss: 6.3079, Train Steps/Sec: 0.90 + 35%|████████████████████████████████████████████████▉ | 16182/45917 [2:56:49<9:07:52, 1.11s/it][2025-04-24 17:28:15] (step=0062100) Train Loss: 6.3468, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████████████████ | 16207/45917 [2:57:17<9:16:37, 1.12s/it][2025-04-24 17:28:42] (step=0062125) Train Loss: 6.2781, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████████████████▏ | 16232/45917 [2:57:45<9:10:31, 1.11s/it][2025-04-24 17:29:10] (step=0062150) Train Loss: 6.3238, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████████████████▏ | 16257/45917 [2:58:12<9:08:18, 1.11s/it][2025-04-24 17:29:38] (step=0062175) Train Loss: 6.3436, Train Steps/Sec: 0.90 + 35%|█████████████████████████████████████████████████▎ | 16282/45917 [2:58:40<9:05:21, 1.10s/it][2025-04-24 17:30:06] (step=0062200) Train Loss: 6.3202, Train Steps/Sec: 0.89 + 36%|█████████████████████████████████████████████████▎ | 16307/45917 [2:59:08<9:15:35, 1.13s/it][2025-04-24 17:30:34] (step=0062225) Train Loss: 6.2816, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████████████████▍ | 16332/45917 [2:59:36<9:05:42, 1.11s/it][2025-04-24 17:31:01] (step=0062250) Train Loss: 6.2967, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████████████████▌ | 16357/45917 [3:00:04<9:04:38, 1.11s/it][2025-04-24 17:31:29] (step=0062275) Train Loss: 6.3058, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████████████████▌ | 16382/45917 [3:00:31<9:01:34, 1.10s/it][2025-04-24 17:31:57] (step=0062300) Train Loss: 6.3351, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████████████████▋ | 16407/45917 [3:00:59<9:17:46, 1.13s/it][2025-04-24 17:32:25] (step=0062325) Train Loss: 6.2873, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████████████████▋ | 16432/45917 [3:01:27<9:05:04, 1.11s/it][2025-04-24 17:32:53] (step=0062350) Train Loss: 6.3109, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████████████████▊ | 16457/45917 [3:01:55<9:07:26, 1.11s/it][2025-04-24 17:33:20] (step=0062375) Train Loss: 6.3090, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████████████████▉ | 16482/45917 [3:02:22<9:00:35, 1.10s/it][2025-04-24 17:33:48] (step=0062400) Train Loss: 6.3421, Train Steps/Sec: 0.90 + 36%|█████████████████████████████████████████████████▉ | 16507/45917 [3:02:50<9:13:22, 1.13s/it][2025-04-24 17:34:16] (step=0062425) Train Loss: 6.2883, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████████████████ | 16532/45917 [3:03:18<9:07:00, 1.12s/it][2025-04-24 17:34:44] (step=0062450) Train Loss: 6.3172, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████████████████ | 16557/45917 [3:03:46<8:59:40, 1.10s/it][2025-04-24 17:35:12] (step=0062475) Train Loss: 6.3300, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████████████████▏ | 16582/45917 [3:04:13<8:53:38, 1.09s/it][2025-04-24 17:35:39] (step=0062500) Train Loss: 6.2917, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████████████████▎ | 16607/45917 [3:04:41<9:09:57, 1.13s/it][2025-04-24 17:36:07] (step=0062525) Train Loss: 6.3215, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████████████████▎ | 16632/45917 [3:05:09<9:01:54, 1.11s/it][2025-04-24 17:36:35] (step=0062550) Train Loss: 6.3159, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████████████████▍ | 16657/45917 [3:05:37<9:00:05, 1.11s/it][2025-04-24 17:37:02] (step=0062575) Train Loss: 6.3100, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████████████████▍ | 16682/45917 [3:06:04<8:52:16, 1.09s/it][2025-04-24 17:37:30] (step=0062600) Train Loss: 6.3172, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████████████████▌ | 16707/45917 [3:06:32<9:09:53, 1.13s/it][2025-04-24 17:37:58] (step=0062625) Train Loss: 6.3015, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████████████████▋ | 16732/45917 [3:07:00<9:02:11, 1.11s/it][2025-04-24 17:38:26] (step=0062650) Train Loss: 6.3335, Train Steps/Sec: 0.90 + 36%|██████████████████████████████████████████████████▋ | 16757/45917 [3:07:28<8:57:18, 1.11s/it][2025-04-24 17:38:53] (step=0062675) Train Loss: 6.3127, Train Steps/Sec: 0.90 + 37%|██████████████████████████████████████████████████▊ | 16782/45917 [3:07:55<8:53:44, 1.10s/it][2025-04-24 17:39:21] (step=0062700) Train Loss: 6.3248, Train Steps/Sec: 0.90 + 37%|██████████████████████████████████████████████████▉ | 16807/45917 [3:08:23<9:02:40, 1.12s/it][2025-04-24 17:39:49] (step=0062725) Train Loss: 6.3022, Train Steps/Sec: 0.91 + 37%|██████████████████████████████████████████████████▉ | 16832/45917 [3:08:51<8:54:07, 1.10s/it][2025-04-24 17:40:16] (step=0062750) Train Loss: 6.3731, Train Steps/Sec: 0.91 + 37%|███████████████████████████████████████████████████ | 16857/45917 [3:09:18<8:55:10, 1.10s/it][2025-04-24 17:40:44] (step=0062775) Train Loss: 6.3288, Train Steps/Sec: 0.91 + 37%|███████████████████████████████████████████████████ | 16882/45917 [3:09:46<8:51:43, 1.10s/it][2025-04-24 17:41:12] (step=0062800) Train Loss: 6.3443, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████████████████▏ | 16907/45917 [3:10:14<9:07:28, 1.13s/it][2025-04-24 17:41:40] (step=0062825) Train Loss: 6.3224, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████████████████▎ | 16932/45917 [3:10:41<8:54:07, 1.11s/it][2025-04-24 17:42:07] (step=0062850) Train Loss: 6.3075, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████████████████▎ | 16957/45917 [3:11:09<8:55:44, 1.11s/it][2025-04-24 17:42:35] (step=0062875) Train Loss: 6.3111, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████████████████▍ | 16982/45917 [3:11:37<8:51:27, 1.10s/it][2025-04-24 17:43:03] (step=0062900) Train Loss: 6.3215, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████████████████▍ | 17007/45917 [3:12:05<9:04:39, 1.13s/it][2025-04-24 17:43:31] (step=0062925) Train Loss: 6.3229, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████████████████▌ | 17032/45917 [3:12:33<8:57:55, 1.12s/it][2025-04-24 17:43:58] (step=0062950) Train Loss: 6.3613, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████████████████▋ | 17057/45917 [3:13:00<8:51:34, 1.11s/it][2025-04-24 17:44:26] (step=0062975) Train Loss: 6.2888, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████████████████▋ | 17082/45917 [3:13:28<8:48:09, 1.10s/it][2025-04-24 17:44:54] (step=0063000) Train Loss: 6.3330, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████████████████▊ | 17107/45917 [3:13:56<8:57:07, 1.12s/it][2025-04-24 17:45:22] (step=0063025) Train Loss: 6.3043, Train Steps/Sec: 0.91 + 37%|███████████████████████████████████████████████████▊ | 17132/45917 [3:14:23<8:58:38, 1.12s/it][2025-04-24 17:45:49] (step=0063050) Train Loss: 6.3200, Train Steps/Sec: 0.90 + 37%|███████████████████████████████████████████████████▉ | 17157/45917 [3:14:51<8:48:24, 1.10s/it][2025-04-24 17:46:17] (step=0063075) Train Loss: 6.2422, Train Steps/Sec: 0.90 + 37%|████████████████████████████████████████████████████ | 17182/45917 [3:15:19<8:47:08, 1.10s/it][2025-04-24 17:46:45] (step=0063100) Train Loss: 6.3217, Train Steps/Sec: 0.90 + 37%|████████████████████████████████████████████████████ | 17207/45917 [3:15:47<9:00:15, 1.13s/it][2025-04-24 17:47:13] (step=0063125) Train Loss: 6.3217, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████████████████▏ | 17232/45917 [3:16:15<8:46:54, 1.10s/it][2025-04-24 17:47:40] (step=0063150) Train Loss: 6.3534, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████████████████▏ | 17257/45917 [3:16:43<8:45:59, 1.10s/it][2025-04-24 17:48:08] (step=0063175) Train Loss: 6.2822, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████████████████▎ | 17282/45917 [3:17:10<8:42:35, 1.09s/it][2025-04-24 17:48:36] (step=0063200) Train Loss: 6.3012, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████████████████▍ | 17307/45917 [3:17:38<8:57:52, 1.13s/it][2025-04-24 17:49:04] (step=0063225) Train Loss: 6.2771, Train Steps/Sec: 0.91 + 38%|████████████████████████████████████████████████████▍ | 17332/45917 [3:18:06<8:46:15, 1.10s/it][2025-04-24 17:49:31] (step=0063250) Train Loss: 6.3590, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████████████████▌ | 17357/45917 [3:18:33<8:46:35, 1.11s/it][2025-04-24 17:49:59] (step=0063275) Train Loss: 6.3455, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████████████████▌ | 17382/45917 [3:19:01<8:41:35, 1.10s/it][2025-04-24 17:50:27] (step=0063300) Train Loss: 6.3276, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████████████████▋ | 17407/45917 [3:19:29<8:56:46, 1.13s/it][2025-04-24 17:50:55] (step=0063325) Train Loss: 6.3164, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████████████████▊ | 17432/45917 [3:19:57<8:46:54, 1.11s/it][2025-04-24 17:51:22] (step=0063350) Train Loss: 6.3235, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████████████████▊ | 17457/45917 [3:20:24<8:49:05, 1.12s/it][2025-04-24 17:51:50] (step=0063375) Train Loss: 6.3216, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████████████████▉ | 17482/45917 [3:20:52<8:38:55, 1.09s/it][2025-04-24 17:52:18] (step=0063400) Train Loss: 6.2960, Train Steps/Sec: 0.90 + 38%|████████████████████████████████████████████████████▉ | 17507/45917 [3:21:20<8:52:06, 1.12s/it][2025-04-24 17:52:45] (step=0063425) Train Loss: 6.3230, Train Steps/Sec: 0.91 + 38%|█████████████████████████████████████████████████████ | 17532/45917 [3:21:47<8:48:02, 1.12s/it][2025-04-24 17:53:13] (step=0063450) Train Loss: 6.3139, Train Steps/Sec: 0.90 + 38%|█████████████████████████████████████████████████████▏ | 17557/45917 [3:22:15<8:37:42, 1.10s/it][2025-04-24 17:53:41] (step=0063475) Train Loss: 6.2889, Train Steps/Sec: 0.90 + 38%|█████████████████████████████████████████████████████▏ | 17582/45917 [3:22:43<8:38:22, 1.10s/it][2025-04-24 17:54:09] (step=0063500) Train Loss: 6.2896, Train Steps/Sec: 0.90 + 38%|█████████████████████████████████████████████████████▎ | 17607/45917 [3:23:11<8:56:12, 1.14s/it][2025-04-24 17:54:37] (step=0063525) Train Loss: 6.3383, Train Steps/Sec: 0.90 + 38%|█████████████████████████████████████████████████████▍ | 17632/45917 [3:23:39<8:43:27, 1.11s/it][2025-04-24 17:55:04] (step=0063550) Train Loss: 6.3114, Train Steps/Sec: 0.90 + 38%|█████████████████████████████████████████████████████▍ | 17657/45917 [3:24:06<8:38:06, 1.10s/it][2025-04-24 17:55:32] (step=0063575) Train Loss: 6.3272, Train Steps/Sec: 0.91 + 39%|█████████████████████████████████████████████████████▌ | 17682/45917 [3:24:34<8:39:55, 1.10s/it][2025-04-24 17:56:00] (step=0063600) Train Loss: 6.3170, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████████████████▌ | 17707/45917 [3:25:02<8:49:01, 1.13s/it][2025-04-24 17:56:27] (step=0063625) Train Loss: 6.3246, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████████████████▋ | 17732/45917 [3:25:29<8:40:45, 1.11s/it][2025-04-24 17:56:55] (step=0063650) Train Loss: 6.2969, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████████████████▊ | 17757/45917 [3:25:57<8:33:53, 1.09s/it][2025-04-24 17:57:23] (step=0063675) Train Loss: 6.3062, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████████████████▊ | 17782/45917 [3:26:25<8:31:23, 1.09s/it][2025-04-24 17:57:51] (step=0063700) Train Loss: 6.3326, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████████████████▉ | 17807/45917 [3:26:53<8:51:28, 1.13s/it][2025-04-24 17:58:19] (step=0063725) Train Loss: 6.3590, Train Steps/Sec: 0.90 + 39%|█████████████████████████████████████████████████████▉ | 17832/45917 [3:27:21<8:36:29, 1.10s/it][2025-04-24 17:58:46] (step=0063750) Train Loss: 6.3301, Train Steps/Sec: 0.90 + 39%|██████████████████████████████████████████████████████ | 17857/45917 [3:27:48<8:36:45, 1.10s/it][2025-04-24 17:59:14] (step=0063775) Train Loss: 6.2811, Train Steps/Sec: 0.90 + 39%|██████████████████████████████████████████████████████▏ | 17882/45917 [3:28:16<8:29:25, 1.09s/it][2025-04-24 17:59:42] (step=0063800) Train Loss: 6.3075, Train Steps/Sec: 0.89 + 39%|██████████████████████████████████████████████████████▏ | 17907/45917 [3:28:44<8:46:16, 1.13s/it][2025-04-24 18:00:10] (step=0063825) Train Loss: 6.3050, Train Steps/Sec: 0.90 + 39%|██████████████████████████████████████████████████████▎ | 17932/45917 [3:29:12<8:37:34, 1.11s/it][2025-04-24 18:00:38] (step=0063850) Train Loss: 6.2993, Train Steps/Sec: 0.90 + 39%|██████████████████████████████████████████████████████▎ | 17957/45917 [3:29:40<8:37:07, 1.11s/it][2025-04-24 18:01:05] (step=0063875) Train Loss: 6.3161, Train Steps/Sec: 0.90 + 39%|██████████████████████████████████████████████████████▍ | 17982/45917 [3:30:07<8:31:41, 1.10s/it][2025-04-24 18:01:33] (step=0063900) Train Loss: 6.2872, Train Steps/Sec: 0.90 + 39%|██████████████████████████████████████████████████████▌ | 18007/45917 [3:30:35<8:45:35, 1.13s/it][2025-04-24 18:02:01] (step=0063925) Train Loss: 6.3456, Train Steps/Sec: 0.90 + 39%|██████████████████████████████████████████████████████▌ | 18032/45917 [3:31:03<8:34:44, 1.11s/it][2025-04-24 18:02:29] (step=0063950) Train Loss: 6.3406, Train Steps/Sec: 0.90 + 39%|██████████████████████████████████████████████████████▋ | 18057/45917 [3:31:31<8:35:26, 1.11s/it][2025-04-24 18:02:57] (step=0063975) Train Loss: 6.2883, Train Steps/Sec: 0.90 + 39%|██████████████████████████████████████████████████████▋ | 18082/45917 [3:31:59<8:29:27, 1.10s/it][2025-04-24 18:03:25] (step=0064000) Train Loss: 6.3845, Train Steps/Sec: 0.90 +[2025-04-24 18:03:25] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 18:03:25] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 18:03:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:59<00:00, 59.73s/it] +[2025-04-24 18:08:11] Finish Eval in 64000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.07s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 18:08:33] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0064000.pt +[2025-04-24 18:08:35] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0062000.pt + 39%|██████████████████████████████████████████████████████▊ | 18107/45917 [3:37:37<8:50:01, 1.14s/it][2025-04-24 18:09:03] (step=0064025) Train Loss: 6.2839, Train Steps/Sec: 0.07 + 39%|██████████████████████████████████████████████████████▉ | 18132/45917 [3:38:05<8:31:28, 1.10s/it][2025-04-24 18:09:30] (step=0064050) Train Loss: 6.3318, Train Steps/Sec: 0.90 + 40%|██████████████████████████████████████████████████████▉ | 18157/45917 [3:38:32<8:32:05, 1.11s/it][2025-04-24 18:09:58] (step=0064075) Train Loss: 6.3614, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████ | 18182/45917 [3:39:00<8:24:31, 1.09s/it][2025-04-24 18:10:26] (step=0064100) Train Loss: 6.3052, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████ | 18207/45917 [3:39:28<8:43:29, 1.13s/it][2025-04-24 18:10:54] (step=0064125) Train Loss: 6.3590, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████▏ | 18232/45917 [3:39:56<8:29:42, 1.10s/it][2025-04-24 18:11:21] (step=0064150) Train Loss: 6.3499, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████▎ | 18257/45917 [3:40:23<8:30:28, 1.11s/it][2025-04-24 18:11:49] (step=0064175) Train Loss: 6.3079, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████▎ | 18282/45917 [3:40:51<8:26:23, 1.10s/it][2025-04-24 18:12:17] (step=0064200) Train Loss: 6.3007, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████▍ | 18307/45917 [3:41:19<8:36:11, 1.12s/it][2025-04-24 18:12:45] (step=0064225) Train Loss: 6.3449, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████▍ | 18332/45917 [3:41:47<8:30:11, 1.11s/it][2025-04-24 18:13:13] (step=0064250) Train Loss: 6.2880, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████▌ | 18357/45917 [3:42:15<8:25:40, 1.10s/it][2025-04-24 18:13:40] (step=0064275) Train Loss: 6.3027, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████▋ | 18382/45917 [3:42:42<8:21:54, 1.09s/it][2025-04-24 18:14:08] (step=0064300) Train Loss: 6.2975, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████▋ | 18407/45917 [3:43:10<8:36:46, 1.13s/it][2025-04-24 18:14:36] (step=0064325) Train Loss: 6.3129, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████▊ | 18432/45917 [3:43:38<8:29:58, 1.11s/it][2025-04-24 18:15:04] (step=0064350) Train Loss: 6.2814, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████▊ | 18457/45917 [3:44:06<8:22:20, 1.10s/it][2025-04-24 18:15:31] (step=0064375) Train Loss: 6.3423, Train Steps/Sec: 0.90 + 40%|███████████████████████████████████████████████████████▉ | 18482/45917 [3:44:33<8:20:06, 1.09s/it][2025-04-24 18:15:59] (step=0064400) Train Loss: 6.3141, Train Steps/Sec: 0.90 + 40%|████████████████████████████████████████████████████████ | 18507/45917 [3:45:01<8:33:10, 1.12s/it][2025-04-24 18:16:27] (step=0064425) Train Loss: 6.3216, Train Steps/Sec: 0.90 + 40%|████████████████████████████████████████████████████████ | 18532/45917 [3:45:29<8:25:52, 1.11s/it][2025-04-24 18:16:55] (step=0064450) Train Loss: 6.3007, Train Steps/Sec: 0.90 + 40%|████████████████████████████████████████████████████████▏ | 18557/45917 [3:45:57<8:23:07, 1.10s/it][2025-04-24 18:17:22] (step=0064475) Train Loss: 6.3726, Train Steps/Sec: 0.90 + 40%|████████████████████████████████████████████████████████▎ | 18582/45917 [3:46:24<8:17:16, 1.09s/it][2025-04-24 18:17:50] (step=0064500) Train Loss: 6.2956, Train Steps/Sec: 0.90 + 41%|████████████████████████████████████████████████████████▎ | 18607/45917 [3:46:52<8:35:40, 1.13s/it][2025-04-24 18:18:18] (step=0064525) Train Loss: 6.3156, Train Steps/Sec: 0.90 + 41%|████████████████████████████████████████████████████████▍ | 18632/45917 [3:47:20<8:23:45, 1.11s/it][2025-04-24 18:18:46] (step=0064550) Train Loss: 6.3656, Train Steps/Sec: 0.90 + 41%|████████████████████████████████████████████████████████▍ | 18657/45917 [3:47:48<8:27:47, 1.12s/it][2025-04-24 18:19:13] (step=0064575) Train Loss: 6.2682, Train Steps/Sec: 0.90 + 41%|████████████████████████████████████████████████████████▌ | 18682/45917 [3:48:15<8:18:18, 1.10s/it][2025-04-24 18:19:41] (step=0064600) Train Loss: 6.3432, Train Steps/Sec: 0.90 + 41%|████████████████████████████████████████████████████████▋ | 18707/45917 [3:48:43<8:30:54, 1.13s/it][2025-04-24 18:20:09] (step=0064625) Train Loss: 6.2993, Train Steps/Sec: 0.90 + 41%|████████████████████████████████████████████████████████▋ | 18732/45917 [3:49:11<8:28:56, 1.12s/it][2025-04-24 18:20:37] (step=0064650) Train Loss: 6.3281, Train Steps/Sec: 0.90 + 41%|████████████████████████████████████████████████████████▊ | 18757/45917 [3:49:39<8:16:59, 1.10s/it][2025-04-24 18:21:04] (step=0064675) Train Loss: 6.3156, Train Steps/Sec: 0.90 + 41%|████████████████████████████████████████████████████████▊ | 18782/45917 [3:50:06<8:19:58, 1.11s/it][2025-04-24 18:21:32] (step=0064700) Train Loss: 6.3169, Train Steps/Sec: 0.90 + 41%|████████████████████████████████████████████████████████▉ | 18807/45917 [3:50:34<8:28:01, 1.12s/it][2025-04-24 18:22:00] (step=0064725) Train Loss: 6.3343, Train Steps/Sec: 0.90 + 41%|█████████████████████████████████████████████████████████ | 18832/45917 [3:51:02<8:24:19, 1.12s/it][2025-04-24 18:22:28] (step=0064750) Train Loss: 6.3509, Train Steps/Sec: 0.90 + 41%|█████████████████████████████████████████████████████████ | 18857/45917 [3:51:30<8:13:52, 1.10s/it][2025-04-24 18:22:55] (step=0064775) Train Loss: 6.3073, Train Steps/Sec: 0.91 + 41%|█████████████████████████████████████████████████████████▏ | 18882/45917 [3:51:57<8:10:39, 1.09s/it][2025-04-24 18:23:23] (step=0064800) Train Loss: 6.2816, Train Steps/Sec: 0.90 + 41%|█████████████████████████████████████████████████████████▏ | 18907/45917 [3:52:25<8:23:49, 1.12s/it][2025-04-24 18:23:51] (step=0064825) Train Loss: 6.3353, Train Steps/Sec: 0.90 + 41%|█████████████████████████████████████████████████████████▎ | 18932/45917 [3:52:53<8:20:08, 1.11s/it][2025-04-24 18:24:19] (step=0064850) Train Loss: 6.3568, Train Steps/Sec: 0.90 + 41%|█████████████████████████████████████████████████████████▍ | 18957/45917 [3:53:21<8:17:34, 1.11s/it][2025-04-24 18:24:46] (step=0064875) Train Loss: 6.2991, Train Steps/Sec: 0.90 + 41%|█████████████████████████████████████████████████████████▍ | 18982/45917 [3:53:48<8:16:32, 1.11s/it][2025-04-24 18:25:14] (step=0064900) Train Loss: 6.2545, Train Steps/Sec: 0.90 + 41%|█████████████████████████████████████████████████████████▌ | 19007/45917 [3:54:16<8:25:54, 1.13s/it][2025-04-24 18:25:42] (step=0064925) Train Loss: 6.3167, Train Steps/Sec: 0.91 + 41%|█████████████████████████████████████████████████████████▌ | 19032/45917 [3:54:44<8:15:36, 1.11s/it][2025-04-24 18:26:09] (step=0064950) Train Loss: 6.2872, Train Steps/Sec: 0.90 + 42%|█████████████████████████████████████████████████████████▋ | 19057/45917 [3:55:12<8:15:52, 1.11s/it][2025-04-24 18:26:37] (step=0064975) Train Loss: 6.3841, Train Steps/Sec: 0.90 + 42%|█████████████████████████████████████████████████████████▊ | 19082/45917 [3:55:39<8:15:57, 1.11s/it][2025-04-24 18:27:05] (step=0065000) Train Loss: 6.2724, Train Steps/Sec: 0.90 + 42%|█████████████████████████████████████████████████████████▊ | 19107/45917 [3:56:07<8:29:00, 1.14s/it][2025-04-24 18:27:33] (step=0065025) Train Loss: 6.3125, Train Steps/Sec: 0.90 + 42%|█████████████████████████████████████████████████████████▉ | 19132/45917 [3:56:40<8:32:41, 1.15s/it][2025-04-24 18:28:06] (step=0065050) Train Loss: 6.3150, Train Steps/Sec: 0.76 + 42%|█████████████████████████████████████████████████████████▉ | 19157/45917 [3:57:08<8:19:07, 1.12s/it][2025-04-24 18:28:34] (step=0065075) Train Loss: 6.3393, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████ | 19182/45917 [3:57:36<8:09:49, 1.10s/it][2025-04-24 18:29:02] (step=0065100) Train Loss: 6.3575, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████▏ | 19207/45917 [3:58:04<8:18:38, 1.12s/it][2025-04-24 18:29:30] (step=0065125) Train Loss: 6.3128, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████▏ | 19232/45917 [3:58:32<8:14:05, 1.11s/it][2025-04-24 18:29:57] (step=0065150) Train Loss: 6.3311, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████▎ | 19257/45917 [3:58:59<8:12:28, 1.11s/it][2025-04-24 18:30:25] (step=0065175) Train Loss: 6.3588, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████▎ | 19282/45917 [3:59:27<8:05:56, 1.09s/it][2025-04-24 18:30:53] (step=0065200) Train Loss: 6.3312, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████▍ | 19307/45917 [3:59:55<8:18:25, 1.12s/it][2025-04-24 18:31:21] (step=0065225) Train Loss: 6.3264, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████▌ | 19332/45917 [4:00:22<8:08:40, 1.10s/it][2025-04-24 18:31:48] (step=0065250) Train Loss: 6.2899, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████▌ | 19357/45917 [4:00:50<8:04:42, 1.09s/it][2025-04-24 18:32:16] (step=0065275) Train Loss: 6.3122, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████▋ | 19382/45917 [4:01:18<8:07:17, 1.10s/it][2025-04-24 18:32:44] (step=0065300) Train Loss: 6.3075, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████▋ | 19407/45917 [4:01:46<8:27:17, 1.15s/it][2025-04-24 18:33:11] (step=0065325) Train Loss: 6.3360, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████▊ | 19432/45917 [4:02:13<8:06:52, 1.10s/it][2025-04-24 18:33:39] (step=0065350) Train Loss: 6.2785, Train Steps/Sec: 0.91 + 42%|██████████████████████████████████████████████████████████▉ | 19457/45917 [4:02:41<8:07:00, 1.10s/it][2025-04-24 18:34:07] (step=0065375) Train Loss: 6.3380, Train Steps/Sec: 0.90 + 42%|██████████████████████████████████████████████████████████▉ | 19482/45917 [4:03:09<8:07:23, 1.11s/it][2025-04-24 18:34:35] (step=0065400) Train Loss: 6.3230, Train Steps/Sec: 0.90 + 42%|███████████████████████████████████████████████████████████ | 19507/45917 [4:03:37<8:16:13, 1.13s/it][2025-04-24 18:35:02] (step=0065425) Train Loss: 6.3219, Train Steps/Sec: 0.90 + 43%|███████████████████████████████████████████████████████████▏ | 19532/45917 [4:04:11<8:08:51, 1.11s/it][2025-04-24 18:35:37] (step=0065450) Train Loss: 6.2963, Train Steps/Sec: 0.73 + 43%|███████████████████████████████████████████████████████████▏ | 19557/45917 [4:04:39<8:05:42, 1.11s/it][2025-04-24 18:36:04] (step=0065475) Train Loss: 6.2665, Train Steps/Sec: 0.91 + 43%|███████████████████████████████████████████████████████████▎ | 19582/45917 [4:05:06<7:55:56, 1.08s/it][2025-04-24 18:36:32] (step=0065500) Train Loss: 6.3055, Train Steps/Sec: 0.90 + 43%|███████████████████████████████████████████████████████████▎ | 19607/45917 [4:05:34<8:15:07, 1.13s/it][2025-04-24 18:37:00] (step=0065525) Train Loss: 6.2850, Train Steps/Sec: 0.90 + 43%|███████████████████████████████████████████████████████████▍ | 19632/45917 [4:06:02<8:06:10, 1.11s/it][2025-04-24 18:37:28] (step=0065550) Train Loss: 6.3445, Train Steps/Sec: 0.90 + 43%|███████████████████████████████████████████████████████████▌ | 19657/45917 [4:06:30<8:00:22, 1.10s/it][2025-04-24 18:37:55] (step=0065575) Train Loss: 6.2991, Train Steps/Sec: 0.90 + 43%|███████████████████████████████████████████████████████████▏ | 19682/45917 [4:07:03<10:00:26, 1.37s/it][2025-04-24 18:38:29] (step=0065600) Train Loss: 6.2821, Train Steps/Sec: 0.75 + 43%|███████████████████████████████████████████████████████████▋ | 19707/45917 [4:07:35<8:13:15, 1.13s/it][2025-04-24 18:39:01] (step=0065625) Train Loss: 6.3374, Train Steps/Sec: 0.77 + 43%|███████████████████████████████████████████████████████████▋ | 19732/45917 [4:08:03<8:05:15, 1.11s/it][2025-04-24 18:39:29] (step=0065650) Train Loss: 6.2943, Train Steps/Sec: 0.90 + 43%|███████████████████████████████████████████████████████████▊ | 19757/45917 [4:08:31<8:00:13, 1.10s/it][2025-04-24 18:39:57] (step=0065675) Train Loss: 6.3175, Train Steps/Sec: 0.91 + 43%|███████████████████████████████████████████████████████████▉ | 19782/45917 [4:09:04<8:00:39, 1.10s/it][2025-04-24 18:40:30] (step=0065700) Train Loss: 6.2695, Train Steps/Sec: 0.76 + 43%|███████████████████████████████████████████████████████████▌ | 19807/45917 [4:09:37<10:03:50, 1.39s/it][2025-04-24 18:41:03] (step=0065725) Train Loss: 6.3218, Train Steps/Sec: 0.76 + 43%|████████████████████████████████████████████████████████████ | 19832/45917 [4:10:05<8:09:18, 1.13s/it][2025-04-24 18:41:30] (step=0065750) Train Loss: 6.2976, Train Steps/Sec: 0.90 + 43%|████████████████████████████████████████████████████████████ | 19857/45917 [4:10:32<7:57:52, 1.10s/it][2025-04-24 18:41:58] (step=0065775) Train Loss: 6.2601, Train Steps/Sec: 0.90 + 43%|████████████████████████████████████████████████████████████▏ | 19882/45917 [4:11:00<7:58:50, 1.10s/it][2025-04-24 18:42:26] (step=0065800) Train Loss: 6.2912, Train Steps/Sec: 0.90 + 43%|████████████████████████████████████████████████████████████▎ | 19907/45917 [4:11:28<8:12:38, 1.14s/it][2025-04-24 18:42:54] (step=0065825) Train Loss: 6.3002, Train Steps/Sec: 0.90 + 43%|████████████████████████████████████████████████████████████▎ | 19932/45917 [4:11:56<7:57:44, 1.10s/it][2025-04-24 18:43:21] (step=0065850) Train Loss: 6.3101, Train Steps/Sec: 0.90 + 43%|████████████████████████████████████████████████████████████▍ | 19957/45917 [4:12:23<8:00:44, 1.11s/it][2025-04-24 18:43:49] (step=0065875) Train Loss: 6.3175, Train Steps/Sec: 0.90 + 44%|████████████████████████████████████████████████████████████▍ | 19982/45917 [4:12:51<7:54:23, 1.10s/it][2025-04-24 18:44:17] (step=0065900) Train Loss: 6.2887, Train Steps/Sec: 0.90 + 44%|████████████████████████████████████████████████████████████▌ | 20007/45917 [4:13:19<8:08:32, 1.13s/it][2025-04-24 18:44:45] (step=0065925) Train Loss: 6.3643, Train Steps/Sec: 0.90 + 44%|████████████████████████████████████████████████████████████▋ | 20032/45917 [4:13:47<7:56:24, 1.10s/it][2025-04-24 18:45:12] (step=0065950) Train Loss: 6.2727, Train Steps/Sec: 0.90 + 44%|████████████████████████████████████████████████████████████▎ | 20057/45917 [4:14:20<10:57:17, 1.53s/it][2025-04-24 18:45:46] (step=0065975) Train Loss: 6.3126, Train Steps/Sec: 0.74 + 44%|████████████████████████████████████████████████████████████▊ | 20082/45917 [4:14:48<7:52:22, 1.10s/it][2025-04-24 18:46:14] (step=0066000) Train Loss: 6.2573, Train Steps/Sec: 0.90 +[2025-04-24 18:46:14] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 18:46:14] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 18:46:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:59<00:00, 59.75s/it] +[2025-04-24 18:51:01] Finish Eval in 66000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.24s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 18:51:23] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0066000.pt +[2025-04-24 18:51:25] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0064000.pt + 44%|████████████████████████████████████████████████████████████▊ | 20107/45917 [4:20:27<8:11:04, 1.14s/it][2025-04-24 18:51:53] (step=0066025) Train Loss: 6.3237, Train Steps/Sec: 0.07 + 44%|████████████████████████████████████████████████████████████▉ | 20132/45917 [4:20:55<7:56:37, 1.11s/it][2025-04-24 18:52:20] (step=0066050) Train Loss: 6.3104, Train Steps/Sec: 0.91 + 44%|█████████████████████████████████████████████████████████████ | 20151/45917 [4:21:16<8:30:34, 1.19s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:646: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 44%|█████████████████████████████████████████████████████████████ | 20157/45917 [4:21:22<7:55:20, 1.11s/it][2025-04-24 18:52:48] (step=0066075) Train Loss: 6.3216, Train Steps/Sec: 0.90 + 44%|█████████████████████████████████████████████████████████████ | 20182/45917 [4:21:50<7:54:34, 1.11s/it][2025-04-24 18:53:16] (step=0066100) Train Loss: 6.3364, Train Steps/Sec: 0.90 + 44%|█████████████████████████████████████████████████████████████▏ | 20207/45917 [4:22:18<7:59:00, 1.12s/it][2025-04-24 18:53:44] (step=0066125) Train Loss: 6.2455, Train Steps/Sec: 0.91 + 44%|█████████████████████████████████████████████████████████████▏ | 20232/45917 [4:22:46<7:54:07, 1.11s/it][2025-04-24 18:54:11] (step=0066150) Train Loss: 6.2264, Train Steps/Sec: 0.90 + 44%|█████████████████████████████████████████████████████████████▎ | 20257/45917 [4:23:13<7:49:21, 1.10s/it][2025-04-24 18:54:39] (step=0066175) Train Loss: 6.3378, Train Steps/Sec: 0.90 + 44%|█████████████████████████████████████████████████████████████▍ | 20282/45917 [4:23:41<7:47:48, 1.09s/it][2025-04-24 18:55:07] (step=0066200) Train Loss: 6.3190, Train Steps/Sec: 0.90 + 44%|█████████████████████████████████████████████████████████████▍ | 20307/45917 [4:24:09<7:59:23, 1.12s/it][2025-04-24 18:55:35] (step=0066225) Train Loss: 6.3046, Train Steps/Sec: 0.90 + 44%|█████████████████████████████████████████████████████████████▌ | 20332/45917 [4:24:37<7:52:55, 1.11s/it][2025-04-24 18:56:02] (step=0066250) Train Loss: 6.3438, Train Steps/Sec: 0.91 + 44%|█████████████████████████████████████████████████████████████▌ | 20357/45917 [4:25:04<7:49:38, 1.10s/it][2025-04-24 18:56:30] (step=0066275) Train Loss: 6.3496, Train Steps/Sec: 0.90 + 44%|█████████████████████████████████████████████████████████████▋ | 20382/45917 [4:25:32<7:43:34, 1.09s/it][2025-04-24 18:56:58] (step=0066300) Train Loss: 6.3372, Train Steps/Sec: 0.90 + 44%|█████████████████████████████████████████████████████████████▊ | 20407/45917 [4:26:00<8:00:49, 1.13s/it][2025-04-24 18:57:26] (step=0066325) Train Loss: 6.3116, Train Steps/Sec: 0.90 + 44%|█████████████████████████████████████████████████████████████▊ | 20432/45917 [4:26:28<7:53:32, 1.11s/it][2025-04-24 18:57:53] (step=0066350) Train Loss: 6.3663, Train Steps/Sec: 0.90 + 45%|█████████████████████████████████████████████████████████████▉ | 20457/45917 [4:26:55<7:48:32, 1.10s/it][2025-04-24 18:58:21] (step=0066375) Train Loss: 6.3475, Train Steps/Sec: 0.90 + 45%|██████████████████████████████████████████████████████████████ | 20482/45917 [4:27:23<7:52:03, 1.11s/it][2025-04-24 18:58:49] (step=0066400) Train Loss: 6.3010, Train Steps/Sec: 0.89 + 45%|██████████████████████████████████████████████████████████████ | 20507/45917 [4:27:51<7:58:35, 1.13s/it][2025-04-24 18:59:17] (step=0066425) Train Loss: 6.3169, Train Steps/Sec: 0.90 + 45%|██████████████████████████████████████████████████████████████▏ | 20532/45917 [4:28:19<7:49:10, 1.11s/it][2025-04-24 18:59:44] (step=0066450) Train Loss: 6.2998, Train Steps/Sec: 0.90 + 45%|██████████████████████████████████████████████████████████████▏ | 20557/45917 [4:28:47<7:50:33, 1.11s/it][2025-04-24 19:00:12] (step=0066475) Train Loss: 6.2896, Train Steps/Sec: 0.89 + 45%|██████████████████████████████████████████████████████████████▎ | 20582/45917 [4:29:14<7:42:33, 1.10s/it][2025-04-24 19:00:40] (step=0066500) Train Loss: 6.2768, Train Steps/Sec: 0.90 + 45%|██████████████████████████████████████████████████████████████▍ | 20607/45917 [4:29:42<7:54:56, 1.13s/it][2025-04-24 19:01:08] (step=0066525) Train Loss: 6.2909, Train Steps/Sec: 0.90 + 45%|██████████████████████████████████████████████████████████████▍ | 20632/45917 [4:30:10<7:46:20, 1.11s/it][2025-04-24 19:01:36] (step=0066550) Train Loss: 6.3574, Train Steps/Sec: 0.90 + 45%|██████████████████████████████████████████████████████████████▌ | 20657/45917 [4:30:38<7:42:55, 1.10s/it][2025-04-24 19:02:03] (step=0066575) Train Loss: 6.3397, Train Steps/Sec: 0.90 + 45%|██████████████████████████████████████████████████████████████▌ | 20682/45917 [4:31:05<7:43:40, 1.10s/it][2025-04-24 19:02:31] (step=0066600) Train Loss: 6.2963, Train Steps/Sec: 0.90 + 45%|██████████████████████████████████████████████████████████████▋ | 20707/45917 [4:31:33<7:55:14, 1.13s/it][2025-04-24 19:02:59] (step=0066625) Train Loss: 6.2739, Train Steps/Sec: 0.90 + 45%|██████████████████████████████████████████████████████████████▊ | 20732/45917 [4:32:01<7:41:10, 1.10s/it][2025-04-24 19:03:26] (step=0066650) Train Loss: 6.2971, Train Steps/Sec: 0.91 + 45%|██████████████████████████████████████████████████████████████▊ | 20757/45917 [4:32:29<7:44:34, 1.11s/it][2025-04-24 19:03:54] (step=0066675) Train Loss: 6.3014, Train Steps/Sec: 0.90 + 45%|██████████████████████████████████████████████████████████████▉ | 20782/45917 [4:32:56<7:35:42, 1.09s/it][2025-04-24 19:04:22] (step=0066700) Train Loss: 6.3006, Train Steps/Sec: 0.90 + 45%|██████████████████████████████████████████████████████████████▉ | 20807/45917 [4:33:24<7:55:16, 1.14s/it][2025-04-24 19:04:50] (step=0066725) Train Loss: 6.2837, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████████████████████ | 20832/45917 [4:33:52<7:46:30, 1.12s/it][2025-04-24 19:05:18] (step=0066750) Train Loss: 6.3068, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████████████████████▏ | 20857/45917 [4:34:20<7:39:09, 1.10s/it][2025-04-24 19:05:45] (step=0066775) Train Loss: 6.3091, Train Steps/Sec: 0.90 + 45%|███████████████████████████████████████████████████████████████▏ | 20882/45917 [4:34:47<7:37:31, 1.10s/it][2025-04-24 19:06:13] (step=0066800) Train Loss: 6.2785, Train Steps/Sec: 0.90 + 46%|███████████████████████████████████████████████████████████████▎ | 20907/45917 [4:35:15<7:50:03, 1.13s/it][2025-04-24 19:06:41] (step=0066825) Train Loss: 6.2653, Train Steps/Sec: 0.90 + 46%|███████████████████████████████████████████████████████████████▎ | 20932/45917 [4:35:43<7:41:30, 1.11s/it][2025-04-24 19:07:09] (step=0066850) Train Loss: 6.3011, Train Steps/Sec: 0.90 + 46%|███████████████████████████████████████████████████████████████▍ | 20957/45917 [4:36:10<7:38:35, 1.10s/it][2025-04-24 19:07:36] (step=0066875) Train Loss: 6.3032, Train Steps/Sec: 0.90 + 46%|███████████████████████████████████████████████████████████████▌ | 20982/45917 [4:36:38<7:38:19, 1.10s/it][2025-04-24 19:08:04] (step=0066900) Train Loss: 6.3323, Train Steps/Sec: 0.90 + 46%|███████████████████████████████████████████████████████████████▌ | 21007/45917 [4:37:06<7:47:59, 1.13s/it][2025-04-24 19:08:32] (step=0066925) Train Loss: 6.3332, Train Steps/Sec: 0.90 + 46%|███████████████████████████████████████████████████████████████▋ | 21032/45917 [4:37:34<7:38:54, 1.11s/it][2025-04-24 19:09:00] (step=0066950) Train Loss: 6.2868, Train Steps/Sec: 0.90 + 46%|███████████████████████████████████████████████████████████████▋ | 21057/45917 [4:38:02<7:39:11, 1.11s/it][2025-04-24 19:09:27] (step=0066975) Train Loss: 6.2807, Train Steps/Sec: 0.90 + 46%|███████████████████████████████████████████████████████████████▊ | 21082/45917 [4:38:29<7:35:20, 1.10s/it][2025-04-24 19:09:55] (step=0067000) Train Loss: 6.3213, Train Steps/Sec: 0.89 + 46%|███████████████████████████████████████████████████████████████▉ | 21107/45917 [4:38:57<7:47:03, 1.13s/it][2025-04-24 19:10:23] (step=0067025) Train Loss: 6.3396, Train Steps/Sec: 0.90 + 46%|███████████████████████████████████████████████████████████████▉ | 21132/45917 [4:39:25<7:39:23, 1.11s/it][2025-04-24 19:10:51] (step=0067050) Train Loss: 6.3091, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████████████████████ | 21157/45917 [4:39:53<7:32:50, 1.10s/it][2025-04-24 19:11:18] (step=0067075) Train Loss: 6.3617, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████████████████████ | 21182/45917 [4:40:20<7:31:07, 1.09s/it][2025-04-24 19:11:46] (step=0067100) Train Loss: 6.2808, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████████████████████▏ | 21207/45917 [4:40:48<7:44:56, 1.13s/it][2025-04-24 19:12:14] (step=0067125) Train Loss: 6.3083, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████████████████████▎ | 21232/45917 [4:41:16<7:37:07, 1.11s/it][2025-04-24 19:12:42] (step=0067150) Train Loss: 6.3327, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████████████████████▎ | 21257/45917 [4:41:44<7:32:35, 1.10s/it][2025-04-24 19:13:09] (step=0067175) Train Loss: 6.3145, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████████████████████▍ | 21282/45917 [4:42:11<7:29:50, 1.10s/it][2025-04-24 19:13:37] (step=0067200) Train Loss: 6.3285, Train Steps/Sec: 0.90 + 46%|████████████████████████████████████████████████████████████████▌ | 21307/45917 [4:42:39<7:41:49, 1.13s/it][2025-04-24 19:14:05] (step=0067225) Train Loss: 6.3363, Train Steps/Sec: 0.91 + 46%|████████████████████████████████████████████████████████████████▌ | 21332/45917 [4:43:07<7:35:52, 1.11s/it][2025-04-24 19:14:32] (step=0067250) Train Loss: 6.3142, Train Steps/Sec: 0.90 + 47%|████████████████████████████████████████████████████████████████▋ | 21357/45917 [4:43:34<7:27:56, 1.09s/it][2025-04-24 19:15:00] (step=0067275) Train Loss: 6.2803, Train Steps/Sec: 0.90 + 47%|████████████████████████████████████████████████████████████████▋ | 21382/45917 [4:44:02<7:27:06, 1.09s/it][2025-04-24 19:15:28] (step=0067300) Train Loss: 6.3404, Train Steps/Sec: 0.90 + 47%|████████████████████████████████████████████████████████████████▊ | 21407/45917 [4:44:30<7:37:52, 1.12s/it][2025-04-24 19:15:56] (step=0067325) Train Loss: 6.2950, Train Steps/Sec: 0.90 + 47%|████████████████████████████████████████████████████████████████▉ | 21432/45917 [4:44:57<7:30:09, 1.10s/it][2025-04-24 19:16:23] (step=0067350) Train Loss: 6.3258, Train Steps/Sec: 0.90 + 47%|████████████████████████████████████████████████████████████████▉ | 21457/45917 [4:45:25<7:30:48, 1.11s/it][2025-04-24 19:16:51] (step=0067375) Train Loss: 6.2837, Train Steps/Sec: 0.91 + 47%|█████████████████████████████████████████████████████████████████ | 21482/45917 [4:45:53<7:27:01, 1.10s/it][2025-04-24 19:17:19] (step=0067400) Train Loss: 6.3646, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████████████████████ | 21507/45917 [4:46:21<7:40:08, 1.13s/it][2025-04-24 19:17:46] (step=0067425) Train Loss: 6.3444, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████████████████████▏ | 21532/45917 [4:46:48<7:28:27, 1.10s/it][2025-04-24 19:18:14] (step=0067450) Train Loss: 6.3482, Train Steps/Sec: 0.91 + 47%|█████████████████████████████████████████████████████████████████▎ | 21557/45917 [4:47:16<7:28:46, 1.11s/it][2025-04-24 19:18:42] (step=0067475) Train Loss: 6.3496, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████████████████████▎ | 21582/45917 [4:47:44<7:21:34, 1.09s/it][2025-04-24 19:19:10] (step=0067500) Train Loss: 6.2976, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████████████████████▍ | 21607/45917 [4:48:11<7:34:52, 1.12s/it][2025-04-24 19:19:37] (step=0067525) Train Loss: 6.3188, Train Steps/Sec: 0.91 + 47%|█████████████████████████████████████████████████████████████████▍ | 21632/45917 [4:48:39<7:26:20, 1.10s/it][2025-04-24 19:20:05] (step=0067550) Train Loss: 6.3099, Train Steps/Sec: 0.91 + 47%|█████████████████████████████████████████████████████████████████▌ | 21657/45917 [4:49:07<7:26:07, 1.10s/it][2025-04-24 19:20:33] (step=0067575) Train Loss: 6.2898, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████████████████████▋ | 21682/45917 [4:49:34<7:21:07, 1.09s/it][2025-04-24 19:21:00] (step=0067600) Train Loss: 6.3043, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████████████████████▋ | 21707/45917 [4:50:02<7:35:16, 1.13s/it][2025-04-24 19:21:28] (step=0067625) Train Loss: 6.3296, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████████████████████▊ | 21732/45917 [4:50:30<7:28:39, 1.11s/it][2025-04-24 19:21:56] (step=0067650) Train Loss: 6.3162, Train Steps/Sec: 0.90 + 47%|█████████████████████████████████████████████████████████████████▊ | 21757/45917 [4:50:58<7:22:46, 1.10s/it][2025-04-24 19:22:23] (step=0067675) Train Loss: 6.3143, Train Steps/Sec: 0.91 + 47%|█████████████████████████████████████████████████████████████████▉ | 21782/45917 [4:51:25<7:23:04, 1.10s/it][2025-04-24 19:22:51] (step=0067700) Train Loss: 6.2603, Train Steps/Sec: 0.90 + 47%|██████████████████████████████████████████████████████████████████ | 21807/45917 [4:51:53<7:33:01, 1.13s/it][2025-04-24 19:23:19] (step=0067725) Train Loss: 6.2656, Train Steps/Sec: 0.91 + 48%|██████████████████████████████████████████████████████████████████ | 21832/45917 [4:52:21<7:22:33, 1.10s/it][2025-04-24 19:23:46] (step=0067750) Train Loss: 6.3023, Train Steps/Sec: 0.91 + 48%|██████████████████████████████████████████████████████████████████▏ | 21857/45917 [4:52:48<7:21:40, 1.10s/it][2025-04-24 19:24:14] (step=0067775) Train Loss: 6.2887, Train Steps/Sec: 0.91 + 48%|██████████████████████████████████████████████████████████████████▏ | 21882/45917 [4:53:16<7:17:46, 1.09s/it][2025-04-24 19:24:42] (step=0067800) Train Loss: 6.2518, Train Steps/Sec: 0.90 + 48%|██████████████████████████████████████████████████████████████████▎ | 21907/45917 [4:53:44<7:29:34, 1.12s/it][2025-04-24 19:25:09] (step=0067825) Train Loss: 6.2711, Train Steps/Sec: 0.91 + 48%|██████████████████████████████████████████████████████████████████▍ | 21932/45917 [4:54:11<7:21:15, 1.10s/it][2025-04-24 19:25:37] (step=0067850) Train Loss: 6.3081, Train Steps/Sec: 0.91 + 48%|██████████████████████████████████████████████████████████████████▍ | 21957/45917 [4:54:39<7:17:49, 1.10s/it][2025-04-24 19:26:04] (step=0067875) Train Loss: 6.2641, Train Steps/Sec: 0.91 + 48%|██████████████████████████████████████████████████████████████████▌ | 21982/45917 [4:55:06<7:15:53, 1.09s/it][2025-04-24 19:26:32] (step=0067900) Train Loss: 6.2905, Train Steps/Sec: 0.90 + 48%|██████████████████████████████████████████████████████████████████▌ | 22007/45917 [4:55:34<7:32:03, 1.13s/it][2025-04-24 19:27:00] (step=0067925) Train Loss: 6.2750, Train Steps/Sec: 0.90 + 48%|██████████████████████████████████████████████████████████████████▋ | 22032/45917 [4:56:02<7:20:50, 1.11s/it][2025-04-24 19:27:28] (step=0067950) Train Loss: 6.2853, Train Steps/Sec: 0.90 + 48%|██████████████████████████████████████████████████████████████████▊ | 22057/45917 [4:56:30<7:18:34, 1.10s/it][2025-04-24 19:27:55] (step=0067975) Train Loss: 6.3313, Train Steps/Sec: 0.90 + 48%|██████████████████████████████████████████████████████████████████▊ | 22082/45917 [4:56:58<7:16:47, 1.10s/it][2025-04-24 19:28:23] (step=0068000) Train Loss: 6.3291, Train Steps/Sec: 0.89 +[2025-04-24 19:28:23] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 19:28:23] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 19:28:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/3 [00:00 +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.59s/it] +[2025-04-24 20:15:22] Finish Eval in 70000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.10s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 20:15:43] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0070000.pt +[2025-04-24 20:15:45] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0068000.pt + 53%|████████████████████████████████████████████████████████████████████████▉ | 24107/45917 [5:44:47<6:54:17, 1.14s/it][2025-04-24 20:16:13] (step=0070025) Train Loss: 6.3638, Train Steps/Sec: 0.07 + 53%|█████████████████████████████████████████████████████████████████████████ | 24132/45917 [5:45:15<6:42:45, 1.11s/it][2025-04-24 20:16:41] (step=0070050) Train Loss: 6.2935, Train Steps/Sec: 0.90 + 53%|█████████████████████████████████████████████████████████████████████████▏ | 24157/45917 [5:45:43<6:39:49, 1.10s/it][2025-04-24 20:17:08] (step=0070075) Train Loss: 6.3478, Train Steps/Sec: 0.90 + 53%|█████████████████████████████████████████████████████████████████████████▏ | 24182/45917 [5:46:10<6:35:53, 1.09s/it][2025-04-24 20:17:36] (step=0070100) Train Loss: 6.3158, Train Steps/Sec: 0.90 + 53%|█████████████████████████████████████████████████████████████████████████▎ | 24207/45917 [5:46:38<7:12:26, 1.20s/it][2025-04-24 20:18:04] (step=0070125) Train Loss: 6.2867, Train Steps/Sec: 0.90 + 53%|█████████████████████████████████████████████████████████████████████████▎ | 24232/45917 [5:47:06<6:44:32, 1.12s/it][2025-04-24 20:18:32] (step=0070150) Train Loss: 6.3214, Train Steps/Sec: 0.90 + 53%|█████████████████████████████████████████████████████████████████████████▍ | 24257/45917 [5:47:34<6:36:15, 1.10s/it][2025-04-24 20:19:00] (step=0070175) Train Loss: 6.3095, Train Steps/Sec: 0.90 + 53%|█████████████████████████████████████████████████████████████████████████▌ | 24282/45917 [5:48:02<6:37:41, 1.10s/it][2025-04-24 20:19:27] (step=0070200) Train Loss: 6.3033, Train Steps/Sec: 0.90 + 53%|█████████████████████████████████████████████████████████████████████████▌ | 24307/45917 [5:48:35<9:59:01, 1.66s/it][2025-04-24 20:20:00] (step=0070225) Train Loss: 6.3238, Train Steps/Sec: 0.76 + 53%|█████████████████████████████████████████████████████████████████████████▋ | 24332/45917 [5:49:02<6:40:20, 1.11s/it][2025-04-24 20:20:28] (step=0070250) Train Loss: 6.2938, Train Steps/Sec: 0.90 + 53%|█████████████████████████████████████████████████████████████████████████▋ | 24357/45917 [5:49:30<6:35:13, 1.10s/it][2025-04-24 20:21:01] (step=0070275) Train Loss: 6.3125, Train Steps/Sec: 0.76 + 53%|█████████████████████████████████████████████████████████████████████████▊ | 24382/45917 [5:50:03<6:34:01, 1.10s/it][2025-04-24 20:21:29] (step=0070300) Train Loss: 6.3208, Train Steps/Sec: 0.90 + 53%|█████████████████████████████████████████████████████████████████████████▉ | 24407/45917 [5:50:31<6:41:21, 1.12s/it][2025-04-24 20:21:56] (step=0070325) Train Loss: 6.3675, Train Steps/Sec: 0.90 + 53%|█████████████████████████████████████████████████████████████████████████▉ | 24432/45917 [5:50:58<6:41:00, 1.12s/it][2025-04-24 20:22:24] (step=0070350) Train Loss: 6.2870, Train Steps/Sec: 0.90 + 53%|██████████████████████████████████████████████████████████████████████████ | 24457/45917 [5:51:38<6:36:24, 1.11s/it][2025-04-24 20:23:03] (step=0070375) Train Loss: 6.2464, Train Steps/Sec: 0.64 + 53%|██████████████████████████████████████████████████████████████████████████ | 24482/45917 [5:52:05<6:31:22, 1.10s/it][2025-04-24 20:23:31] (step=0070400) Train Loss: 6.3510, Train Steps/Sec: 0.90 + 53%|█████████████████████████████████████████████████████████████████████████▋ | 24507/45917 [5:52:38<13:08:32, 2.21s/it][2025-04-24 20:24:04] (step=0070425) Train Loss: 6.3140, Train Steps/Sec: 0.76 + 53%|██████████████████████████████████████████████████████████████████████████▎ | 24532/45917 [5:53:06<6:34:58, 1.11s/it][2025-04-24 20:24:32] (step=0070450) Train Loss: 6.3172, Train Steps/Sec: 0.90 + 53%|██████████████████████████████████████████████████████████████████████████▎ | 24557/45917 [5:53:34<6:34:07, 1.11s/it][2025-04-24 20:24:59] (step=0070475) Train Loss: 6.3038, Train Steps/Sec: 0.90 + 54%|██████████████████████████████████████████████████████████████████████████▍ | 24582/45917 [5:54:01<6:29:16, 1.09s/it][2025-04-24 20:25:27] (step=0070500) Train Loss: 6.3081, Train Steps/Sec: 0.90 + 54%|██████████████████████████████████████████████████████████████████████████▍ | 24607/45917 [5:54:29<6:38:29, 1.12s/it][2025-04-24 20:25:55] (step=0070525) Train Loss: 6.3045, Train Steps/Sec: 0.90 + 54%|██████████████████████████████████████████████████████████████████████████▌ | 24632/45917 [5:54:57<6:35:45, 1.12s/it][2025-04-24 20:26:23] (step=0070550) Train Loss: 6.2952, Train Steps/Sec: 0.90 + 54%|██████████████████████████████████████████████████████████████████████████▋ | 24657/45917 [5:55:25<6:30:15, 1.10s/it][2025-04-24 20:26:51] (step=0070575) Train Loss: 6.2746, Train Steps/Sec: 0.90 + 54%|██████████████████████████████████████████████████████████████████████████▋ | 24682/45917 [5:55:52<6:28:28, 1.10s/it][2025-04-24 20:27:18] (step=0070600) Train Loss: 6.2969, Train Steps/Sec: 0.90 + 54%|██████████████████████████████████████████████████████████████████████████▊ | 24707/45917 [5:56:20<6:36:03, 1.12s/it][2025-04-24 20:27:46] (step=0070625) Train Loss: 6.3603, Train Steps/Sec: 0.90 + 54%|██████████████████████████████████████████████████████████████████████████▊ | 24732/45917 [5:56:48<6:32:54, 1.11s/it][2025-04-24 20:28:14] (step=0070650) Train Loss: 6.2986, Train Steps/Sec: 0.90 + 54%|██████████████████████████████████████████████████████████████████████████▉ | 24757/45917 [5:57:16<6:27:17, 1.10s/it][2025-04-24 20:28:41] (step=0070675) Train Loss: 6.2894, Train Steps/Sec: 0.91 + 54%|███████████████████████████████████████████████████████████████████████████ | 24782/45917 [5:57:43<6:26:38, 1.10s/it][2025-04-24 20:29:09] (step=0070700) Train Loss: 6.2988, Train Steps/Sec: 0.89 + 54%|███████████████████████████████████████████████████████████████████████████ | 24807/45917 [5:58:11<6:35:52, 1.13s/it][2025-04-24 20:29:37] (step=0070725) Train Loss: 6.2767, Train Steps/Sec: 0.90 + 54%|███████████████████████████████████████████████████████████████████████████▏ | 24832/45917 [5:58:39<6:30:13, 1.11s/it][2025-04-24 20:30:05] (step=0070750) Train Loss: 6.3087, Train Steps/Sec: 0.90 + 54%|███████████████████████████████████████████████████████████████████████████▏ | 24857/45917 [5:59:07<6:27:13, 1.10s/it][2025-04-24 20:30:33] (step=0070775) Train Loss: 6.3322, Train Steps/Sec: 0.90 + 54%|███████████████████████████████████████████████████████████████████████████▎ | 24882/45917 [5:59:34<6:22:55, 1.09s/it][2025-04-24 20:31:00] (step=0070800) Train Loss: 6.3074, Train Steps/Sec: 0.90 + 54%|███████████████████████████████████████████████████████████████████████████▍ | 24907/45917 [6:00:02<6:35:27, 1.13s/it][2025-04-24 20:31:28] (step=0070825) Train Loss: 6.2802, Train Steps/Sec: 0.90 + 54%|███████████████████████████████████████████████████████████████████████████▍ | 24932/45917 [6:00:30<6:29:25, 1.11s/it][2025-04-24 20:31:56] (step=0070850) Train Loss: 6.3167, Train Steps/Sec: 0.90 + 54%|███████████████████████████████████████████████████████████████████████████▌ | 24957/45917 [6:00:58<6:25:47, 1.10s/it][2025-04-24 20:32:24] (step=0070875) Train Loss: 6.2784, Train Steps/Sec: 0.90 + 54%|███████████████████████████████████████████████████████████████████████████▋ | 24982/45917 [6:01:26<6:25:54, 1.11s/it][2025-04-24 20:32:51] (step=0070900) Train Loss: 6.3193, Train Steps/Sec: 0.90 + 54%|███████████████████████████████████████████████████████████████████████████▋ | 25007/45917 [6:01:53<6:32:39, 1.13s/it][2025-04-24 20:33:19] (step=0070925) Train Loss: 6.2956, Train Steps/Sec: 0.90 + 55%|███████████████████████████████████████████████████████████████████████████▊ | 25032/45917 [6:02:21<6:23:09, 1.10s/it][2025-04-24 20:33:47] (step=0070950) Train Loss: 6.2763, Train Steps/Sec: 0.90 + 55%|███████████████████████████████████████████████████████████████████████████▊ | 25057/45917 [6:02:49<6:25:30, 1.11s/it][2025-04-24 20:34:15] (step=0070975) Train Loss: 6.3284, Train Steps/Sec: 0.91 + 55%|███████████████████████████████████████████████████████████████████████████▉ | 25082/45917 [6:03:16<6:18:36, 1.09s/it][2025-04-24 20:34:42] (step=0071000) Train Loss: 6.3265, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████ | 25107/45917 [6:03:44<6:31:15, 1.13s/it][2025-04-24 20:35:10] (step=0071025) Train Loss: 6.3116, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████ | 25132/45917 [6:04:12<6:24:49, 1.11s/it][2025-04-24 20:35:38] (step=0071050) Train Loss: 6.3067, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████▏ | 25157/45917 [6:04:40<6:21:19, 1.10s/it][2025-04-24 20:36:06] (step=0071075) Train Loss: 6.3440, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████▏ | 25182/45917 [6:05:08<6:20:04, 1.10s/it][2025-04-24 20:36:33] (step=0071100) Train Loss: 6.3288, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████▎ | 25207/45917 [6:05:35<6:28:41, 1.13s/it][2025-04-24 20:37:01] (step=0071125) Train Loss: 6.3157, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████▍ | 25232/45917 [6:06:03<6:23:38, 1.11s/it][2025-04-24 20:37:29] (step=0071150) Train Loss: 6.3131, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████▍ | 25257/45917 [6:06:31<6:19:32, 1.10s/it][2025-04-24 20:37:57] (step=0071175) Train Loss: 6.2783, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████▌ | 25282/45917 [6:07:04<6:15:37, 1.09s/it][2025-04-24 20:38:30] (step=0071200) Train Loss: 6.2539, Train Steps/Sec: 0.75 + 55%|████████████████████████████████████████████████████████████████████████████▌ | 25307/45917 [6:07:32<6:24:24, 1.12s/it][2025-04-24 20:38:58] (step=0071225) Train Loss: 6.2633, Train Steps/Sec: 0.91 + 55%|████████████████████████████████████████████████████████████████████████████▋ | 25332/45917 [6:08:00<6:20:11, 1.11s/it][2025-04-24 20:39:25] (step=0071250) Train Loss: 6.3146, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████▊ | 25357/45917 [6:08:27<6:18:43, 1.11s/it][2025-04-24 20:39:53] (step=0071275) Train Loss: 6.3180, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████▊ | 25382/45917 [6:08:55<6:13:25, 1.09s/it][2025-04-24 20:40:21] (step=0071300) Train Loss: 6.2932, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████▉ | 25407/45917 [6:09:23<6:25:35, 1.13s/it][2025-04-24 20:40:49] (step=0071325) Train Loss: 6.3309, Train Steps/Sec: 0.90 + 55%|████████████████████████████████████████████████████████████████████████████▉ | 25432/45917 [6:09:51<6:18:30, 1.11s/it][2025-04-24 20:41:16] (step=0071350) Train Loss: 6.3025, Train Steps/Sec: 0.91 + 55%|█████████████████████████████████████████████████████████████████████████████ | 25457/45917 [6:10:18<6:14:22, 1.10s/it][2025-04-24 20:41:44] (step=0071375) Train Loss: 6.2968, Train Steps/Sec: 0.90 + 55%|█████████████████████████████████████████████████████████████████████████████▏ | 25482/45917 [6:10:46<6:12:03, 1.09s/it][2025-04-24 20:42:12] (step=0071400) Train Loss: 6.3173, Train Steps/Sec: 0.90 + 56%|█████████████████████████████████████████████████████████████████████████████▏ | 25507/45917 [6:11:14<6:22:21, 1.12s/it][2025-04-24 20:42:40] (step=0071425) Train Loss: 6.3232, Train Steps/Sec: 0.90 + 56%|█████████████████████████████████████████████████████████████████████████████▎ | 25532/45917 [6:11:41<6:15:30, 1.11s/it][2025-04-24 20:43:07] (step=0071450) Train Loss: 6.2974, Train Steps/Sec: 0.91 + 56%|█████████████████████████████████████████████████████████████████████████████▎ | 25557/45917 [6:12:09<6:12:28, 1.10s/it][2025-04-24 20:43:35] (step=0071475) Train Loss: 6.2991, Train Steps/Sec: 0.91 + 56%|█████████████████████████████████████████████████████████████████████████████▍ | 25582/45917 [6:12:37<6:14:16, 1.10s/it][2025-04-24 20:44:03] (step=0071500) Train Loss: 6.2467, Train Steps/Sec: 0.90 + 56%|█████████████████████████████████████████████████████████████████████████████▌ | 25607/45917 [6:13:05<6:22:32, 1.13s/it][2025-04-24 20:44:30] (step=0071525) Train Loss: 6.3446, Train Steps/Sec: 0.90 + 56%|█████████████████████████████████████████████████████████████████████████████▌ | 25632/45917 [6:13:32<6:10:41, 1.10s/it][2025-04-24 20:44:58] (step=0071550) Train Loss: 6.2539, Train Steps/Sec: 0.91 + 56%|█████████████████████████████████████████████████████████████████████████████▋ | 25657/45917 [6:14:00<6:10:51, 1.10s/it][2025-04-24 20:45:26] (step=0071575) Train Loss: 6.3225, Train Steps/Sec: 0.90 + 56%|█████████████████████████████████████████████████████████████████████████████▋ | 25682/45917 [6:14:27<6:08:18, 1.09s/it][2025-04-24 20:45:53] (step=0071600) Train Loss: 6.2962, Train Steps/Sec: 0.90 + 56%|█████████████████████████████████████████████████████████████████████████████▊ | 25707/45917 [6:14:55<6:21:37, 1.13s/it][2025-04-24 20:46:21] (step=0071625) Train Loss: 6.2930, Train Steps/Sec: 0.90 + 56%|█████████████████████████████████████████████████████████████████████████████▉ | 25732/45917 [6:15:23<6:13:41, 1.11s/it][2025-04-24 20:46:49] (step=0071650) Train Loss: 6.3070, Train Steps/Sec: 0.90 + 56%|█████████████████████████████████████████████████████████████████████████████▉ | 25757/45917 [6:15:51<6:09:13, 1.10s/it][2025-04-24 20:47:16] (step=0071675) Train Loss: 6.2677, Train Steps/Sec: 0.91 + 56%|██████████████████████████████████████████████████████████████████████████████ | 25782/45917 [6:16:18<6:06:31, 1.09s/it][2025-04-24 20:47:44] (step=0071700) Train Loss: 6.2733, Train Steps/Sec: 0.90 + 56%|██████████████████████████████████████████████████████████████████████████████ | 25807/45917 [6:16:46<6:19:44, 1.13s/it][2025-04-24 20:48:12] (step=0071725) Train Loss: 6.2963, Train Steps/Sec: 0.90 + 56%|██████████████████████████████████████████████████████████████████████████████▏ | 25832/45917 [6:17:14<6:11:02, 1.11s/it][2025-04-24 20:48:40] (step=0071750) Train Loss: 6.3088, Train Steps/Sec: 0.90 + 56%|██████████████████████████████████████████████████████████████████████████████▎ | 25857/45917 [6:17:42<6:09:50, 1.11s/it][2025-04-24 20:49:07] (step=0071775) Train Loss: 6.2860, Train Steps/Sec: 0.90 + 56%|██████████████████████████████████████████████████████████████████████████████▎ | 25882/45917 [6:18:09<6:05:10, 1.09s/it][2025-04-24 20:49:35] (step=0071800) Train Loss: 6.3005, Train Steps/Sec: 0.90 + 56%|██████████████████████████████████████████████████████████████████████████████▍ | 25907/45917 [6:18:37<6:13:35, 1.12s/it][2025-04-24 20:50:03] (step=0071825) Train Loss: 6.2473, Train Steps/Sec: 0.90 + 56%|██████████████████████████████████████████████████████████████████████████████▌ | 25932/45917 [6:19:05<6:07:18, 1.10s/it][2025-04-24 20:50:31] (step=0071850) Train Loss: 6.3002, Train Steps/Sec: 0.90 + 57%|██████████████████████████████████████████████████████████████████████████████▌ | 25957/45917 [6:19:32<6:04:19, 1.10s/it][2025-04-24 20:50:58] (step=0071875) Train Loss: 6.3358, Train Steps/Sec: 0.90 + 57%|██████████████████████████████████████████████████████████████████████████████▋ | 25982/45917 [6:20:00<6:03:30, 1.09s/it][2025-04-24 20:51:26] (step=0071900) Train Loss: 6.3282, Train Steps/Sec: 0.90 + 57%|██████████████████████████████████████████████████████████████████████████████▋ | 26007/45917 [6:20:28<6:12:09, 1.12s/it][2025-04-24 20:51:54] (step=0071925) Train Loss: 6.2850, Train Steps/Sec: 0.90 + 57%|██████████████████████████████████████████████████████████████████████████████▊ | 26032/45917 [6:20:56<6:06:25, 1.11s/it][2025-04-24 20:52:21] (step=0071950) Train Loss: 6.3039, Train Steps/Sec: 0.90 + 57%|██████████████████████████████████████████████████████████████████████████████▉ | 26057/45917 [6:21:23<6:04:14, 1.10s/it][2025-04-24 20:52:49] (step=0071975) Train Loss: 6.3195, Train Steps/Sec: 0.90 + 57%|██████████████████████████████████████████████████████████████████████████████▉ | 26082/45917 [6:21:51<6:00:30, 1.09s/it][2025-04-24 20:53:17] (step=0072000) Train Loss: 6.3039, Train Steps/Sec: 0.90 +[2025-04-24 20:53:17] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 20:53:17] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 20:53:17] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.93s/it] +[2025-04-24 20:58:01] Finish Eval in 72000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.48s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 20:58:21] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0072000.pt +[2025-04-24 20:58:23] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0070000.pt + 57%|███████████████████████████████████████████████████████████████████████████████ | 26107/45917 [6:27:26<6:17:46, 1.14s/it][2025-04-24 20:58:52] (step=0072025) Train Loss: 6.2627, Train Steps/Sec: 0.07 + 57%|███████████████████████████████████████████████████████████████████████████████ | 26132/45917 [6:27:54<6:07:48, 1.12s/it][2025-04-24 20:59:19] (step=0072050) Train Loss: 6.2974, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████████████████████████▏ | 26157/45917 [6:28:21<6:02:12, 1.10s/it][2025-04-24 20:59:47] (step=0072075) Train Loss: 6.3281, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████████████████████████▎ | 26182/45917 [6:28:49<6:01:14, 1.10s/it][2025-04-24 21:00:15] (step=0072100) Train Loss: 6.2769, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████████████████████████▎ | 26207/45917 [6:29:17<6:10:11, 1.13s/it][2025-04-24 21:00:43] (step=0072125) Train Loss: 6.3154, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████████████████████████▍ | 26232/45917 [6:29:45<6:04:10, 1.11s/it][2025-04-24 21:01:10] (step=0072150) Train Loss: 6.2704, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████████████████████████▍ | 26257/45917 [6:30:12<6:02:01, 1.10s/it][2025-04-24 21:01:38] (step=0072175) Train Loss: 6.3267, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████████████████████████▌ | 26282/45917 [6:30:40<5:58:15, 1.09s/it][2025-04-24 21:02:06] (step=0072200) Train Loss: 6.3470, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████████████████████████▋ | 26307/45917 [6:31:08<6:06:10, 1.12s/it][2025-04-24 21:02:33] (step=0072225) Train Loss: 6.3171, Train Steps/Sec: 0.91 + 57%|███████████████████████████████████████████████████████████████████████████████▋ | 26332/45917 [6:31:35<6:05:21, 1.12s/it][2025-04-24 21:03:01] (step=0072250) Train Loss: 6.2998, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████████████████████████▊ | 26357/45917 [6:32:03<5:58:22, 1.10s/it][2025-04-24 21:03:29] (step=0072275) Train Loss: 6.2902, Train Steps/Sec: 0.90 + 57%|███████████████████████████████████████████████████████████████████████████████▊ | 26382/45917 [6:32:31<5:57:33, 1.10s/it][2025-04-24 21:03:57] (step=0072300) Train Loss: 6.2798, Train Steps/Sec: 0.90 + 58%|███████████████████████████████████████████████████████████████████████████████▉ | 26407/45917 [6:32:59<6:04:52, 1.12s/it][2025-04-24 21:04:25] (step=0072325) Train Loss: 6.3313, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████ | 26432/45917 [6:33:26<5:58:57, 1.11s/it][2025-04-24 21:04:52] (step=0072350) Train Loss: 6.2533, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████ | 26457/45917 [6:33:54<5:59:19, 1.11s/it][2025-04-24 21:05:20] (step=0072375) Train Loss: 6.2766, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▏ | 26482/45917 [6:34:22<5:56:32, 1.10s/it][2025-04-24 21:05:48] (step=0072400) Train Loss: 6.2788, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▏ | 26507/45917 [6:34:50<6:07:26, 1.14s/it][2025-04-24 21:06:16] (step=0072425) Train Loss: 6.3500, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▎ | 26532/45917 [6:35:18<5:57:51, 1.11s/it][2025-04-24 21:06:43] (step=0072450) Train Loss: 6.3715, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▍ | 26557/45917 [6:35:45<5:57:48, 1.11s/it][2025-04-24 21:07:11] (step=0072475) Train Loss: 6.3105, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▍ | 26582/45917 [6:36:13<5:59:29, 1.12s/it][2025-04-24 21:07:39] (step=0072500) Train Loss: 6.2876, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▌ | 26607/45917 [6:36:41<5:59:49, 1.12s/it][2025-04-24 21:08:07] (step=0072525) Train Loss: 6.3012, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▌ | 26632/45917 [6:37:09<5:57:52, 1.11s/it][2025-04-24 21:08:34] (step=0072550) Train Loss: 6.3021, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▋ | 26657/45917 [6:37:36<5:53:34, 1.10s/it][2025-04-24 21:09:02] (step=0072575) Train Loss: 6.3326, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▊ | 26682/45917 [6:38:04<5:49:27, 1.09s/it][2025-04-24 21:09:30] (step=0072600) Train Loss: 6.2812, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▊ | 26707/45917 [6:38:32<6:00:43, 1.13s/it][2025-04-24 21:09:57] (step=0072625) Train Loss: 6.3193, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▉ | 26732/45917 [6:39:00<5:56:08, 1.11s/it][2025-04-24 21:10:25] (step=0072650) Train Loss: 6.3219, Train Steps/Sec: 0.90 + 58%|████████████████████████████████████████████████████████████████████████████████▉ | 26757/45917 [6:39:27<5:54:21, 1.11s/it][2025-04-24 21:10:53] (step=0072675) Train Loss: 6.2810, Train Steps/Sec: 0.90 + 58%|█████████████████████████████████████████████████████████████████████████████████ | 26782/45917 [6:39:55<5:50:44, 1.10s/it][2025-04-24 21:11:21] (step=0072700) Train Loss: 6.2577, Train Steps/Sec: 0.90 + 58%|█████████████████████████████████████████████████████████████████████████████████▏ | 26807/45917 [6:40:23<5:59:39, 1.13s/it][2025-04-24 21:11:49] (step=0072725) Train Loss: 6.3139, Train Steps/Sec: 0.90 + 58%|█████████████████████████████████████████████████████████████████████████████████▏ | 26832/45917 [6:40:51<5:51:10, 1.10s/it][2025-04-24 21:12:16] (step=0072750) Train Loss: 6.2752, Train Steps/Sec: 0.90 + 58%|█████████████████████████████████████████████████████████████████████████████████▎ | 26857/45917 [6:41:18<5:49:21, 1.10s/it][2025-04-24 21:12:44] (step=0072775) Train Loss: 6.3029, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████████████████████████▍ | 26882/45917 [6:41:46<5:49:46, 1.10s/it][2025-04-24 21:13:12] (step=0072800) Train Loss: 6.3039, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████████████████████████▍ | 26907/45917 [6:42:14<5:56:23, 1.12s/it][2025-04-24 21:13:40] (step=0072825) Train Loss: 6.3105, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████████████████████████▌ | 26932/45917 [6:42:42<5:49:26, 1.10s/it][2025-04-24 21:14:07] (step=0072850) Train Loss: 6.3014, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████████████████████████▌ | 26957/45917 [6:43:09<5:48:17, 1.10s/it][2025-04-24 21:14:35] (step=0072875) Train Loss: 6.2858, Train Steps/Sec: 0.91 + 59%|█████████████████████████████████████████████████████████████████████████████████▋ | 26982/45917 [6:43:37<5:49:09, 1.11s/it][2025-04-24 21:15:03] (step=0072900) Train Loss: 6.3278, Train Steps/Sec: 0.89 + 59%|█████████████████████████████████████████████████████████████████████████████████▊ | 27007/45917 [6:44:05<5:56:28, 1.13s/it][2025-04-24 21:15:31] (step=0072925) Train Loss: 6.2768, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████████████████████████▊ | 27032/45917 [6:44:33<5:48:56, 1.11s/it][2025-04-24 21:15:58] (step=0072950) Train Loss: 6.3274, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████████████████████████▉ | 27057/45917 [6:45:00<5:51:04, 1.12s/it][2025-04-24 21:16:26] (step=0072975) Train Loss: 6.3171, Train Steps/Sec: 0.90 + 59%|█████████████████████████████████████████████████████████████████████████████████▉ | 27082/45917 [6:45:28<5:44:10, 1.10s/it][2025-04-24 21:16:54] (step=0073000) Train Loss: 6.3085, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████████████████████████ | 27107/45917 [6:45:56<5:54:38, 1.13s/it][2025-04-24 21:17:22] (step=0073025) Train Loss: 6.2625, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████████████████████████▏ | 27132/45917 [6:46:24<5:48:41, 1.11s/it][2025-04-24 21:17:50] (step=0073050) Train Loss: 6.3074, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████████████████████████▏ | 27157/45917 [6:46:52<5:46:11, 1.11s/it][2025-04-24 21:18:17] (step=0073075) Train Loss: 6.3343, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████████████████████████▎ | 27182/45917 [6:47:19<5:37:33, 1.08s/it][2025-04-24 21:18:45] (step=0073100) Train Loss: 6.3196, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████████████████████████▎ | 27207/45917 [6:47:47<5:52:12, 1.13s/it][2025-04-24 21:19:13] (step=0073125) Train Loss: 6.3235, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████████████████████████▍ | 27232/45917 [6:48:15<5:45:46, 1.11s/it][2025-04-24 21:19:41] (step=0073150) Train Loss: 6.2531, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████████████████████████▌ | 27257/45917 [6:48:43<5:42:12, 1.10s/it][2025-04-24 21:20:08] (step=0073175) Train Loss: 6.3130, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████████████████████████▌ | 27282/45917 [6:49:10<5:38:46, 1.09s/it][2025-04-24 21:20:36] (step=0073200) Train Loss: 6.2940, Train Steps/Sec: 0.90 + 59%|██████████████████████████████████████████████████████████████████████████████████▋ | 27307/45917 [6:49:38<5:50:11, 1.13s/it][2025-04-24 21:21:04] (step=0073225) Train Loss: 6.3163, Train Steps/Sec: 0.90 + 60%|██████████████████████████████████████████████████████████████████████████████████▋ | 27332/45917 [6:50:06<5:44:26, 1.11s/it][2025-04-24 21:21:32] (step=0073250) Train Loss: 6.2478, Train Steps/Sec: 0.91 + 60%|██████████████████████████████████████████████████████████████████████████████████▊ | 27357/45917 [6:50:33<5:42:51, 1.11s/it][2025-04-24 21:21:59] (step=0073275) Train Loss: 6.3184, Train Steps/Sec: 0.90 + 60%|██████████████████████████████████████████████████████████████████████████████████▉ | 27382/45917 [6:51:01<5:38:06, 1.09s/it][2025-04-24 21:22:27] (step=0073300) Train Loss: 6.2868, Train Steps/Sec: 0.90 + 60%|██████████████████████████████████████████████████████████████████████████████████▉ | 27407/45917 [6:51:29<5:46:44, 1.12s/it][2025-04-24 21:22:55] (step=0073325) Train Loss: 6.2726, Train Steps/Sec: 0.91 + 60%|███████████████████████████████████████████████████████████████████████████████████ | 27432/45917 [6:51:56<5:39:56, 1.10s/it][2025-04-24 21:23:22] (step=0073350) Train Loss: 6.2554, Train Steps/Sec: 0.91 + 60%|███████████████████████████████████████████████████████████████████████████████████ | 27457/45917 [6:52:24<5:41:46, 1.11s/it][2025-04-24 21:23:50] (step=0073375) Train Loss: 6.3001, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████████████████████████▏ | 27482/45917 [6:52:52<5:38:36, 1.10s/it][2025-04-24 21:24:18] (step=0073400) Train Loss: 6.3416, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████████████████████████▎ | 27507/45917 [6:53:20<5:46:16, 1.13s/it][2025-04-24 21:24:45] (step=0073425) Train Loss: 6.3121, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████████████████████████▎ | 27532/45917 [6:53:47<5:35:21, 1.09s/it][2025-04-24 21:25:13] (step=0073450) Train Loss: 6.2953, Train Steps/Sec: 0.91 + 60%|███████████████████████████████████████████████████████████████████████████████████▍ | 27557/45917 [6:54:15<5:38:57, 1.11s/it][2025-04-24 21:25:41] (step=0073475) Train Loss: 6.3075, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████████████████████████▍ | 27582/45917 [6:54:43<5:34:45, 1.10s/it][2025-04-24 21:26:09] (step=0073500) Train Loss: 6.2918, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████████████████████████▌ | 27607/45917 [6:55:11<5:43:34, 1.13s/it][2025-04-24 21:26:36] (step=0073525) Train Loss: 6.3229, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████████████████████████▋ | 27632/45917 [6:55:38<5:37:13, 1.11s/it][2025-04-24 21:27:04] (step=0073550) Train Loss: 6.2982, Train Steps/Sec: 0.91 + 60%|███████████████████████████████████████████████████████████████████████████████████▋ | 27657/45917 [6:56:06<5:37:15, 1.11s/it][2025-04-24 21:27:32] (step=0073575) Train Loss: 6.2901, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████████████████████████▊ | 27682/45917 [6:56:34<5:33:20, 1.10s/it][2025-04-24 21:28:00] (step=0073600) Train Loss: 6.2934, Train Steps/Sec: 0.89 + 60%|███████████████████████████████████████████████████████████████████████████████████▊ | 27707/45917 [6:57:02<5:41:22, 1.12s/it][2025-04-24 21:28:27] (step=0073625) Train Loss: 6.3271, Train Steps/Sec: 0.90 + 60%|███████████████████████████████████████████████████████████████████████████████████▉ | 27732/45917 [6:57:29<5:38:45, 1.12s/it][2025-04-24 21:28:55] (step=0073650) Train Loss: 6.2632, Train Steps/Sec: 0.91 + 60%|████████████████████████████████████████████████████████████████████████████████████ | 27757/45917 [6:57:57<5:30:08, 1.09s/it][2025-04-24 21:29:23] (step=0073675) Train Loss: 6.2859, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████████████████████████ | 27782/45917 [6:58:25<5:31:24, 1.10s/it][2025-04-24 21:29:51] (step=0073700) Train Loss: 6.3038, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████████████████████████▏ | 27807/45917 [6:58:53<5:41:44, 1.13s/it][2025-04-24 21:30:18] (step=0073725) Train Loss: 6.2971, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████████████████████████▎ | 27832/45917 [6:59:20<5:33:58, 1.11s/it][2025-04-24 21:30:46] (step=0073750) Train Loss: 6.2687, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████████████████████████▎ | 27857/45917 [6:59:48<5:30:00, 1.10s/it][2025-04-24 21:31:14] (step=0073775) Train Loss: 6.3468, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████████████████████████▍ | 27882/45917 [7:00:16<5:30:20, 1.10s/it][2025-04-24 21:31:42] (step=0073800) Train Loss: 6.2747, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████████████████████████▍ | 27907/45917 [7:00:44<5:36:03, 1.12s/it][2025-04-24 21:32:09] (step=0073825) Train Loss: 6.3135, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████████████████████████▌ | 27932/45917 [7:01:11<5:30:22, 1.10s/it][2025-04-24 21:32:37] (step=0073850) Train Loss: 6.3306, Train Steps/Sec: 0.91 + 61%|████████████████████████████████████████████████████████████████████████████████████▋ | 27957/45917 [7:01:39<5:26:58, 1.09s/it][2025-04-24 21:33:05] (step=0073875) Train Loss: 6.3177, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████████████████████████▋ | 27982/45917 [7:02:07<5:27:18, 1.09s/it][2025-04-24 21:33:32] (step=0073900) Train Loss: 6.3511, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████████████████████████▊ | 28007/45917 [7:02:35<5:37:33, 1.13s/it][2025-04-24 21:34:00] (step=0073925) Train Loss: 6.2958, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████████████████████████▊ | 28032/45917 [7:03:02<5:31:41, 1.11s/it][2025-04-24 21:34:28] (step=0073950) Train Loss: 6.2872, Train Steps/Sec: 0.90 + 61%|████████████████████████████████████████████████████████████████████████████████████▉ | 28057/45917 [7:03:30<5:28:48, 1.10s/it][2025-04-24 21:34:56] (step=0073975) Train Loss: 6.3096, Train Steps/Sec: 0.90 + 61%|█████████████████████████████████████████████████████████████████████████████████████ | 28082/45917 [7:03:58<5:25:56, 1.10s/it][2025-04-24 21:35:24] (step=0074000) Train Loss: 6.2681, Train Steps/Sec: 0.90 +[2025-04-24 21:35:24] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 21:35:24] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 21:35:24] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:00<00:00, 60.25s/it] +[2025-04-24 21:40:12] Finish Eval in 74000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:00<00:00, 59.60s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 21:40:33] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0074000.pt +[2025-04-24 21:40:35] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0072000.pt + 61%|█████████████████████████████████████████████████████████████████████████████████████ | 28107/45917 [7:09:38<5:38:25, 1.14s/it][2025-04-24 21:41:03] (step=0074025) Train Loss: 6.2745, Train Steps/Sec: 0.07 + 61%|█████████████████████████████████████████████████████████████████████████████████████▏ | 28132/45917 [7:10:05<5:26:53, 1.10s/it][2025-04-24 21:41:31] (step=0074050) Train Loss: 6.2923, Train Steps/Sec: 0.90 + 61%|█████████████████████████████████████████████████████████████████████████████████████▏ | 28157/45917 [7:10:33<5:25:35, 1.10s/it][2025-04-24 21:41:59] (step=0074075) Train Loss: 6.2699, Train Steps/Sec: 0.90 + 61%|█████████████████████████████████████████████████████████████████████████████████████▎ | 28182/45917 [7:11:01<5:22:16, 1.09s/it][2025-04-24 21:42:27] (step=0074100) Train Loss: 6.2758, Train Steps/Sec: 0.90 + 61%|█████████████████████████████████████████████████████████████████████████████████████▍ | 28207/45917 [7:11:28<5:31:00, 1.12s/it][2025-04-24 21:42:54] (step=0074125) Train Loss: 6.2748, Train Steps/Sec: 0.90 + 61%|█████████████████████████████████████████████████████████████████████████████████████▍ | 28232/45917 [7:11:56<5:26:04, 1.11s/it][2025-04-24 21:43:22] (step=0074150) Train Loss: 6.3117, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████████████████████████▌ | 28257/45917 [7:12:24<5:26:27, 1.11s/it][2025-04-24 21:43:50] (step=0074175) Train Loss: 6.3042, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████████████████████████▌ | 28282/45917 [7:12:52<5:22:50, 1.10s/it][2025-04-24 21:44:17] (step=0074200) Train Loss: 6.3236, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████████████████████████▋ | 28307/45917 [7:13:19<5:29:37, 1.12s/it][2025-04-24 21:44:45] (step=0074225) Train Loss: 6.3315, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████████████████████████▊ | 28332/45917 [7:13:47<5:23:18, 1.10s/it][2025-04-24 21:45:13] (step=0074250) Train Loss: 6.3021, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████████████████████████▊ | 28357/45917 [7:14:15<5:22:40, 1.10s/it][2025-04-24 21:45:40] (step=0074275) Train Loss: 6.2806, Train Steps/Sec: 0.91 + 62%|█████████████████████████████████████████████████████████████████████████████████████▉ | 28382/45917 [7:14:42<5:18:15, 1.09s/it][2025-04-24 21:46:08] (step=0074300) Train Loss: 6.2484, Train Steps/Sec: 0.90 + 62%|█████████████████████████████████████████████████████████████████████████████████████▉ | 28407/45917 [7:15:10<5:31:23, 1.14s/it][2025-04-24 21:46:36] (step=0074325) Train Loss: 6.3228, Train Steps/Sec: 0.90 + 62%|██████████████████████████████████████████████████████████████████████████████████████ | 28432/45917 [7:15:38<5:26:51, 1.12s/it][2025-04-24 21:47:04] (step=0074350) Train Loss: 6.3701, Train Steps/Sec: 0.90 + 62%|██████████████████████████████████████████████████████████████████████████████████████▏ | 28457/45917 [7:16:06<5:20:24, 1.10s/it][2025-04-24 21:47:31] (step=0074375) Train Loss: 6.3521, Train Steps/Sec: 0.90 + 62%|██████████████████████████████████████████████████████████████████████████████████████▏ | 28482/45917 [7:16:33<5:19:13, 1.10s/it][2025-04-24 21:47:59] (step=0074400) Train Loss: 6.3185, Train Steps/Sec: 0.90 + 62%|██████████████████████████████████████████████████████████████████████████████████████▎ | 28507/45917 [7:17:01<5:29:54, 1.14s/it][2025-04-24 21:48:27] (step=0074425) Train Loss: 6.3603, Train Steps/Sec: 0.90 + 62%|██████████████████████████████████████████████████████████████████████████████████████▎ | 28532/45917 [7:17:29<5:21:23, 1.11s/it][2025-04-24 21:48:55] (step=0074450) Train Loss: 6.3421, Train Steps/Sec: 0.91 + 62%|██████████████████████████████████████████████████████████████████████████████████████▍ | 28557/45917 [7:17:56<5:20:17, 1.11s/it][2025-04-24 21:49:22] (step=0074475) Train Loss: 6.2940, Train Steps/Sec: 0.91 + 62%|██████████████████████████████████████████████████████████████████████████████████████▌ | 28582/45917 [7:18:24<5:18:55, 1.10s/it][2025-04-24 21:49:50] (step=0074500) Train Loss: 6.3286, Train Steps/Sec: 0.89 + 62%|██████████████████████████████████████████████████████████████████████████████████████▌ | 28607/45917 [7:18:52<5:25:09, 1.13s/it][2025-04-24 21:50:18] (step=0074525) Train Loss: 6.3292, Train Steps/Sec: 0.90 + 62%|██████████████████████████████████████████████████████████████████████████████████████▋ | 28632/45917 [7:19:20<5:18:34, 1.11s/it][2025-04-24 21:50:46] (step=0074550) Train Loss: 6.3545, Train Steps/Sec: 0.90 + 62%|██████████████████████████████████████████████████████████████████████████████████████▊ | 28657/45917 [7:19:48<5:18:23, 1.11s/it][2025-04-24 21:51:13] (step=0074575) Train Loss: 6.3021, Train Steps/Sec: 0.90 + 62%|██████████████████████████████████████████████████████████████████████████████████████▊ | 28682/45917 [7:20:15<5:14:44, 1.10s/it][2025-04-24 21:51:41] (step=0074600) Train Loss: 6.3218, Train Steps/Sec: 0.90 + 63%|██████████████████████████████████████████████████████████████████████████████████████▉ | 28707/45917 [7:20:43<5:23:20, 1.13s/it][2025-04-24 21:52:14] (step=0074625) Train Loss: 6.3184, Train Steps/Sec: 0.77 + 63%|██████████████████████████████████████████████████████████████████████████████████████▉ | 28732/45917 [7:21:16<5:18:09, 1.11s/it][2025-04-24 21:52:42] (step=0074650) Train Loss: 6.3253, Train Steps/Sec: 0.90 + 63%|███████████████████████████████████████████████████████████████████████████████████████ | 28757/45917 [7:21:44<5:17:33, 1.11s/it][2025-04-24 21:53:09] (step=0074675) Train Loss: 6.3496, Train Steps/Sec: 0.90 + 63%|███████████████████████████████████████████████████████████████████████████████████████▏ | 28782/45917 [7:22:11<5:12:02, 1.09s/it][2025-04-24 21:53:37] (step=0074700) Train Loss: 6.3238, Train Steps/Sec: 0.90 + 63%|███████████████████████████████████████████████████████████████████████████████████████▏ | 28807/45917 [7:22:39<5:20:33, 1.12s/it][2025-04-24 21:54:05] (step=0074725) Train Loss: 6.3160, Train Steps/Sec: 0.90 + 63%|███████████████████████████████████████████████████████████████████████████████████████▎ | 28832/45917 [7:23:07<5:13:44, 1.10s/it][2025-04-24 21:54:33] (step=0074750) Train Loss: 6.3521, Train Steps/Sec: 0.90 + 63%|███████████████████████████████████████████████████████████████████████████████████████▎ | 28857/45917 [7:23:35<5:12:07, 1.10s/it][2025-04-24 21:55:00] (step=0074775) Train Loss: 6.3766, Train Steps/Sec: 0.90 + 63%|███████████████████████████████████████████████████████████████████████████████████████▍ | 28882/45917 [7:24:02<5:11:23, 1.10s/it][2025-04-24 21:55:28] (step=0074800) Train Loss: 6.3149, Train Steps/Sec: 0.90 + 63%|███████████████████████████████████████████████████████████████████████████████████████▌ | 28907/45917 [7:24:30<5:19:56, 1.13s/it][2025-04-24 21:55:56] (step=0074825) Train Loss: 6.3149, Train Steps/Sec: 0.91 + 63%|███████████████████████████████████████████████████████████████████████████████████████▌ | 28932/45917 [7:24:58<5:14:01, 1.11s/it][2025-04-24 21:56:23] (step=0074850) Train Loss: 6.2692, Train Steps/Sec: 0.90 + 63%|███████████████████████████████████████████████████████████████████████████████████████▋ | 28957/45917 [7:25:25<5:10:38, 1.10s/it][2025-04-24 21:56:51] (step=0074875) Train Loss: 6.2832, Train Steps/Sec: 0.90 + 63%|███████████████████████████████████████████████████████████████████████████████████████▋ | 28982/45917 [7:25:53<5:11:49, 1.10s/it][2025-04-24 21:57:19] (step=0074900) Train Loss: 6.3569, Train Steps/Sec: 0.90 + 63%|███████████████████████████████████████████████████████████████████████████████████████▊ | 29007/45917 [7:26:21<5:17:05, 1.13s/it][2025-04-24 21:57:47] (step=0074925) Train Loss: 6.3231, Train Steps/Sec: 0.90 + 63%|███████████████████████████████████████████████████████████████████████████████████████▉ | 29032/45917 [7:26:54<8:35:38, 1.83s/it][2025-04-24 21:58:19] (step=0074950) Train Loss: 6.3269, Train Steps/Sec: 0.77 + 63%|███████████████████████████████████████████████████████████████████████████████████████▉ | 29057/45917 [7:27:21<5:09:24, 1.10s/it][2025-04-24 21:58:47] (step=0074975) Train Loss: 6.3047, Train Steps/Sec: 0.90 + 63%|████████████████████████████████████████████████████████████████████████████████████████ | 29082/45917 [7:27:49<5:05:02, 1.09s/it][2025-04-24 21:59:15] (step=0075000) Train Loss: 6.3085, Train Steps/Sec: 0.90 + 63%|████████████████████████████████████████████████████████████████████████████████████████ | 29107/45917 [7:28:22<5:29:12, 1.18s/it][2025-04-24 21:59:47] (step=0075025) Train Loss: 6.3017, Train Steps/Sec: 0.76 + 63%|████████████████████████████████████████████████████████████████████████████████████████▏ | 29132/45917 [7:28:54<5:09:51, 1.11s/it][2025-04-24 22:00:20] (step=0075050) Train Loss: 6.2947, Train Steps/Sec: 0.77 + 63%|████████████████████████████████████████████████████████████████████████████████████████▎ | 29157/45917 [7:29:22<5:06:10, 1.10s/it][2025-04-24 22:00:48] (step=0075075) Train Loss: 6.2741, Train Steps/Sec: 0.90 + 64%|████████████████████████████████████████████████████████████████████████████████████████▎ | 29182/45917 [7:29:50<5:07:21, 1.10s/it][2025-04-24 22:01:16] (step=0075100) Train Loss: 6.3193, Train Steps/Sec: 0.90 + 64%|████████████████████████████████████████████████████████████████████████████████████████▍ | 29207/45917 [7:30:23<5:13:48, 1.13s/it][2025-04-24 22:01:49] (step=0075125) Train Loss: 6.3241, Train Steps/Sec: 0.76 + 64%|████████████████████████████████████████████████████████████████████████████████████████▍ | 29232/45917 [7:30:56<5:14:27, 1.13s/it][2025-04-24 22:02:22] (step=0075150) Train Loss: 6.3013, Train Steps/Sec: 0.77 + 64%|████████████████████████████████████████████████████████████████████████████████████████▌ | 29257/45917 [7:31:24<5:07:40, 1.11s/it][2025-04-24 22:02:49] (step=0075175) Train Loss: 6.3260, Train Steps/Sec: 0.90 + 64%|████████████████████████████████████████████████████████████████████████████████████████▋ | 29282/45917 [7:31:51<5:05:53, 1.10s/it][2025-04-24 22:03:17] (step=0075200) Train Loss: 6.3278, Train Steps/Sec: 0.90 + 64%|████████████████████████████████████████████████████████████████████████████████████████▋ | 29307/45917 [7:32:19<5:10:19, 1.12s/it][2025-04-24 22:03:45] (step=0075225) Train Loss: 6.2289, Train Steps/Sec: 0.90 + 64%|████████████████████████████████████████████████████████████████████████████████████████▊ | 29332/45917 [7:32:47<5:04:01, 1.10s/it][2025-04-24 22:04:13] (step=0075250) Train Loss: 6.3501, Train Steps/Sec: 0.90 + 64%|████████████████████████████████████████████████████████████████████████████████████████▊ | 29357/45917 [7:33:14<5:04:03, 1.10s/it][2025-04-24 22:04:40] (step=0075275) Train Loss: 6.3070, Train Steps/Sec: 0.90 + 64%|████████████████████████████████████████████████████████████████████████████████████████▉ | 29382/45917 [7:33:42<5:01:06, 1.09s/it][2025-04-24 22:05:08] (step=0075300) Train Loss: 6.2892, Train Steps/Sec: 0.90 + 64%|█████████████████████████████████████████████████████████████████████████████████████████ | 29407/45917 [7:34:10<5:08:25, 1.12s/it][2025-04-24 22:05:36] (step=0075325) Train Loss: 6.3169, Train Steps/Sec: 0.90 + 64%|█████████████████████████████████████████████████████████████████████████████████████████ | 29432/45917 [7:34:38<5:05:34, 1.11s/it][2025-04-24 22:06:03] (step=0075350) Train Loss: 6.3049, Train Steps/Sec: 0.90 + 64%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 29457/45917 [7:35:05<5:03:26, 1.11s/it][2025-04-24 22:06:31] (step=0075375) Train Loss: 6.2868, Train Steps/Sec: 0.90 + 64%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 29482/45917 [7:35:33<5:02:24, 1.10s/it][2025-04-24 22:06:59] (step=0075400) Train Loss: 6.3019, Train Steps/Sec: 0.90 + 64%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 29507/45917 [7:36:01<5:08:28, 1.13s/it][2025-04-24 22:07:27] (step=0075425) Train Loss: 6.3012, Train Steps/Sec: 0.90 + 64%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 29532/45917 [7:36:29<5:03:46, 1.11s/it][2025-04-24 22:07:55] (step=0075450) Train Loss: 6.2838, Train Steps/Sec: 0.90 + 64%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 29557/45917 [7:36:56<5:00:13, 1.10s/it][2025-04-24 22:08:22] (step=0075475) Train Loss: 6.3376, Train Steps/Sec: 0.90 + 64%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 29582/45917 [7:37:24<4:59:49, 1.10s/it][2025-04-24 22:08:50] (step=0075500) Train Loss: 6.2986, Train Steps/Sec: 0.90 + 64%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 29607/45917 [7:37:52<5:04:45, 1.12s/it][2025-04-24 22:09:18] (step=0075525) Train Loss: 6.2431, Train Steps/Sec: 0.91 + 65%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 29632/45917 [7:38:20<5:04:41, 1.12s/it][2025-04-24 22:09:46] (step=0075550) Train Loss: 6.3022, Train Steps/Sec: 0.90 + 65%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 29657/45917 [7:38:48<4:59:09, 1.10s/it][2025-04-24 22:10:13] (step=0075575) Train Loss: 6.2903, Train Steps/Sec: 0.90 + 65%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 29682/45917 [7:39:15<4:53:50, 1.09s/it][2025-04-24 22:10:41] (step=0075600) Train Loss: 6.3237, Train Steps/Sec: 0.90 + 65%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 29707/45917 [7:39:43<5:03:23, 1.12s/it][2025-04-24 22:11:09] (step=0075625) Train Loss: 6.3362, Train Steps/Sec: 0.91 + 65%|██████████████████████████████████████████████████████████████████████████████████████████ | 29732/45917 [7:40:11<4:59:55, 1.11s/it][2025-04-24 22:11:37] (step=0075650) Train Loss: 6.2655, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████ | 29757/45917 [7:40:38<4:52:14, 1.09s/it][2025-04-24 22:12:04] (step=0075675) Train Loss: 6.3133, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 29782/45917 [7:41:06<4:53:48, 1.09s/it][2025-04-24 22:12:32] (step=0075700) Train Loss: 6.2951, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 29807/45917 [7:41:34<5:05:14, 1.14s/it][2025-04-24 22:13:00] (step=0075725) Train Loss: 6.3471, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 29832/45917 [7:42:02<4:59:21, 1.12s/it][2025-04-24 22:13:28] (step=0075750) Train Loss: 6.3075, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 29857/45917 [7:42:29<4:55:36, 1.10s/it][2025-04-24 22:13:55] (step=0075775) Train Loss: 6.3028, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 29882/45917 [7:42:57<4:52:42, 1.10s/it][2025-04-24 22:14:23] (step=0075800) Train Loss: 6.2136, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 29907/45917 [7:43:25<5:00:52, 1.13s/it][2025-04-24 22:14:51] (step=0075825) Train Loss: 6.3236, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 29932/45917 [7:43:53<4:56:48, 1.11s/it][2025-04-24 22:15:18] (step=0075850) Train Loss: 6.2577, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 29957/45917 [7:44:20<4:52:02, 1.10s/it][2025-04-24 22:15:46] (step=0075875) Train Loss: 6.3237, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 29982/45917 [7:44:48<4:49:00, 1.09s/it][2025-04-24 22:16:14] (step=0075900) Train Loss: 6.3072, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 30007/45917 [7:45:16<5:01:23, 1.14s/it][2025-04-24 22:16:42] (step=0075925) Train Loss: 6.3252, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 30032/45917 [7:45:43<4:53:09, 1.11s/it][2025-04-24 22:17:09] (step=0075950) Train Loss: 6.3051, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 30057/45917 [7:46:11<4:55:11, 1.12s/it][2025-04-24 22:17:37] (step=0075975) Train Loss: 6.3279, Train Steps/Sec: 0.90 + 66%|███████████████████████████████████████████████████████████████████████████████████████████ | 30082/45917 [7:46:39<4:49:35, 1.10s/it][2025-04-24 22:18:05] (step=0076000) Train Loss: 6.3243, Train Steps/Sec: 0.90 +[2025-04-24 22:18:05] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 22:18:05] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 22:18:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 59.11s/it] +[2025-04-24 22:22:50] Finish Eval in 76000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.61s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 22:23:12] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0076000.pt +[2025-04-24 22:23:14] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0074000.pt + 66%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 30107/45917 [7:52:17<5:04:03, 1.15s/it][2025-04-24 22:23:43] (step=0076025) Train Loss: 6.2960, Train Steps/Sec: 0.07 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 30132/45917 [7:52:45<4:52:21, 1.11s/it][2025-04-24 22:24:10] (step=0076050) Train Loss: 6.3248, Train Steps/Sec: 0.90 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 30157/45917 [7:53:12<4:49:04, 1.10s/it][2025-04-24 22:24:38] (step=0076075) Train Loss: 6.2920, Train Steps/Sec: 0.90 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 30182/45917 [7:53:40<4:47:44, 1.10s/it][2025-04-24 22:25:06] (step=0076100) Train Loss: 6.3132, Train Steps/Sec: 0.90 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 30207/45917 [7:54:08<4:52:49, 1.12s/it][2025-04-24 22:25:34] (step=0076125) Train Loss: 6.3547, Train Steps/Sec: 0.90 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 30232/45917 [7:54:36<4:56:31, 1.13s/it][2025-04-24 22:26:01] (step=0076150) Train Loss: 6.2921, Train Steps/Sec: 0.90 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 30257/45917 [7:55:03<4:49:24, 1.11s/it][2025-04-24 22:26:29] (step=0076175) Train Loss: 6.3275, Train Steps/Sec: 0.90 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 30282/45917 [7:55:31<4:44:24, 1.09s/it][2025-04-24 22:26:57] (step=0076200) Train Loss: 6.2448, Train Steps/Sec: 0.90 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 30307/45917 [7:55:59<4:52:56, 1.13s/it][2025-04-24 22:27:25] (step=0076225) Train Loss: 6.2372, Train Steps/Sec: 0.91 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 30332/45917 [7:56:26<4:48:17, 1.11s/it][2025-04-24 22:27:52] (step=0076250) Train Loss: 6.3004, Train Steps/Sec: 0.91 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 30357/45917 [7:56:54<4:45:27, 1.10s/it][2025-04-24 22:28:20] (step=0076275) Train Loss: 6.3131, Train Steps/Sec: 0.90 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 30382/45917 [7:57:22<4:43:50, 1.10s/it][2025-04-24 22:28:48] (step=0076300) Train Loss: 6.3087, Train Steps/Sec: 0.90 + 66%|████████████████████████████████████████████████████████████████████████████████████████████ | 30407/45917 [7:57:50<4:50:48, 1.12s/it][2025-04-24 22:29:15] (step=0076325) Train Loss: 6.2633, Train Steps/Sec: 0.90 + 66%|████████████████████████████████████████████████████████████████████████████████████████████ | 30432/45917 [7:58:17<4:46:44, 1.11s/it][2025-04-24 22:29:43] (step=0076350) Train Loss: 6.2984, Train Steps/Sec: 0.91 + 66%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 30457/45917 [7:58:51<6:00:16, 1.40s/it][2025-04-24 22:30:17] (step=0076375) Train Loss: 6.3410, Train Steps/Sec: 0.74 + 66%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 30482/45917 [7:59:19<4:41:56, 1.10s/it][2025-04-24 22:30:44] (step=0076400) Train Loss: 6.2952, Train Steps/Sec: 0.90 + 66%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 30507/45917 [7:59:46<4:51:57, 1.14s/it][2025-04-24 22:31:12] (step=0076425) Train Loss: 6.2965, Train Steps/Sec: 0.90 + 66%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 30532/45917 [8:00:14<4:43:24, 1.11s/it][2025-04-24 22:31:40] (step=0076450) Train Loss: 6.2975, Train Steps/Sec: 0.91 + 67%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 30557/45917 [8:00:42<4:41:25, 1.10s/it][2025-04-24 22:32:07] (step=0076475) Train Loss: 6.2784, Train Steps/Sec: 0.90 + 67%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 30582/45917 [8:01:09<4:39:34, 1.09s/it][2025-04-24 22:32:35] (step=0076500) Train Loss: 6.3102, Train Steps/Sec: 0.90 + 67%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 30607/45917 [8:01:37<4:46:55, 1.12s/it][2025-04-24 22:33:03] (step=0076525) Train Loss: 6.3320, Train Steps/Sec: 0.90 + 67%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 30632/45917 [8:02:05<4:42:53, 1.11s/it][2025-04-24 22:33:31] (step=0076550) Train Loss: 6.3202, Train Steps/Sec: 0.90 + 67%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 30657/45917 [8:02:33<4:41:38, 1.11s/it][2025-04-24 22:33:59] (step=0076575) Train Loss: 6.2958, Train Steps/Sec: 0.90 + 67%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 30682/45917 [8:03:01<4:38:13, 1.10s/it][2025-04-24 22:34:26] (step=0076600) Train Loss: 6.2984, Train Steps/Sec: 0.90 + 67%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 30707/45917 [8:03:28<4:48:49, 1.14s/it][2025-04-24 22:34:54] (step=0076625) Train Loss: 6.2667, Train Steps/Sec: 0.90 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████ | 30732/45917 [8:03:56<4:39:36, 1.10s/it][2025-04-24 22:35:22] (step=0076650) Train Loss: 6.3031, Train Steps/Sec: 0.90 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████ | 30757/45917 [8:04:24<4:40:35, 1.11s/it][2025-04-24 22:35:50] (step=0076675) Train Loss: 6.2950, Train Steps/Sec: 0.90 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 30782/45917 [8:04:52<4:35:38, 1.09s/it][2025-04-24 22:36:18] (step=0076700) Train Loss: 6.3268, Train Steps/Sec: 0.90 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 30807/45917 [8:05:19<4:41:44, 1.12s/it][2025-04-24 22:36:45] (step=0076725) Train Loss: 6.3045, Train Steps/Sec: 0.90 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 30832/45917 [8:05:47<4:36:34, 1.10s/it][2025-04-24 22:37:13] (step=0076750) Train Loss: 6.3179, Train Steps/Sec: 0.91 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 30857/45917 [8:06:15<4:37:04, 1.10s/it][2025-04-24 22:37:40] (step=0076775) Train Loss: 6.2889, Train Steps/Sec: 0.90 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 30882/45917 [8:06:42<4:33:06, 1.09s/it][2025-04-24 22:38:08] (step=0076800) Train Loss: 6.3053, Train Steps/Sec: 0.90 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 30907/45917 [8:07:10<4:38:02, 1.11s/it][2025-04-24 22:38:36] (step=0076825) Train Loss: 6.2841, Train Steps/Sec: 0.91 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 30932/45917 [8:07:38<4:32:00, 1.09s/it][2025-04-24 22:39:03] (step=0076850) Train Loss: 6.3237, Train Steps/Sec: 0.91 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 30957/45917 [8:08:05<4:33:48, 1.10s/it][2025-04-24 22:39:31] (step=0076875) Train Loss: 6.2881, Train Steps/Sec: 0.90 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 30982/45917 [8:08:33<4:33:20, 1.10s/it][2025-04-24 22:39:59] (step=0076900) Train Loss: 6.2906, Train Steps/Sec: 0.90 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 31007/45917 [8:09:01<4:42:26, 1.14s/it][2025-04-24 22:40:27] (step=0076925) Train Loss: 6.3433, Train Steps/Sec: 0.90 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 31032/45917 [8:09:28<4:33:24, 1.10s/it][2025-04-24 22:40:54] (step=0076950) Train Loss: 6.2805, Train Steps/Sec: 0.91 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████ | 31057/45917 [8:09:56<4:32:51, 1.10s/it][2025-04-24 22:41:22] (step=0076975) Train Loss: 6.3019, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████ | 31082/45917 [8:10:24<4:31:53, 1.10s/it][2025-04-24 22:41:50] (step=0077000) Train Loss: 6.2659, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 31107/45917 [8:10:52<4:36:58, 1.12s/it][2025-04-24 22:42:17] (step=0077025) Train Loss: 6.2931, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 31132/45917 [8:11:19<4:33:30, 1.11s/it][2025-04-24 22:42:45] (step=0077050) Train Loss: 6.3218, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 31157/45917 [8:11:47<4:29:35, 1.10s/it][2025-04-24 22:43:13] (step=0077075) Train Loss: 6.2964, Train Steps/Sec: 0.91 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 31182/45917 [8:12:15<4:28:40, 1.09s/it][2025-04-24 22:43:41] (step=0077100) Train Loss: 6.3347, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 31207/45917 [8:12:43<4:35:53, 1.13s/it][2025-04-24 22:44:08] (step=0077125) Train Loss: 6.3067, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 31232/45917 [8:13:10<4:32:13, 1.11s/it][2025-04-24 22:44:36] (step=0077150) Train Loss: 6.3069, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 31257/45917 [8:13:38<4:29:26, 1.10s/it][2025-04-24 22:45:04] (step=0077175) Train Loss: 6.2616, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 31282/45917 [8:14:06<4:25:40, 1.09s/it][2025-04-24 22:45:31] (step=0077200) Train Loss: 6.3413, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 31307/45917 [8:14:33<4:34:37, 1.13s/it][2025-04-24 22:45:59] (step=0077225) Train Loss: 6.2545, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 31332/45917 [8:15:01<4:28:49, 1.11s/it][2025-04-24 22:46:27] (step=0077250) Train Loss: 6.3122, Train Steps/Sec: 0.91 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 31357/45917 [8:15:29<4:29:09, 1.11s/it][2025-04-24 22:46:54] (step=0077275) Train Loss: 6.3178, Train Steps/Sec: 0.90 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 31382/45917 [8:15:56<4:24:34, 1.09s/it][2025-04-24 22:47:22] (step=0077300) Train Loss: 6.2742, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████████████████████████████ | 31407/45917 [8:16:24<4:31:21, 1.12s/it][2025-04-24 22:47:50] (step=0077325) Train Loss: 6.3125, Train Steps/Sec: 0.90 + 68%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 31432/45917 [8:16:52<4:26:17, 1.10s/it][2025-04-24 22:48:18] (step=0077350) Train Loss: 6.2814, Train Steps/Sec: 0.90 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 31457/45917 [8:17:19<4:24:49, 1.10s/it][2025-04-24 22:48:45] (step=0077375) Train Loss: 6.3218, Train Steps/Sec: 0.91 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 31482/45917 [8:17:47<4:24:06, 1.10s/it][2025-04-24 22:49:13] (step=0077400) Train Loss: 6.3238, Train Steps/Sec: 0.90 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 31507/45917 [8:18:15<4:29:50, 1.12s/it][2025-04-24 22:49:41] (step=0077425) Train Loss: 6.2766, Train Steps/Sec: 0.90 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 31532/45917 [8:18:43<4:26:30, 1.11s/it][2025-04-24 22:50:09] (step=0077450) Train Loss: 6.3266, Train Steps/Sec: 0.90 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 31557/45917 [8:19:11<4:24:54, 1.11s/it][2025-04-24 22:50:36] (step=0077475) Train Loss: 6.3077, Train Steps/Sec: 0.90 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 31582/45917 [8:19:38<4:22:18, 1.10s/it][2025-04-24 22:51:04] (step=0077500) Train Loss: 6.3222, Train Steps/Sec: 0.90 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 31607/45917 [8:20:06<4:28:18, 1.12s/it][2025-04-24 22:51:32] (step=0077525) Train Loss: 6.2936, Train Steps/Sec: 0.90 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 31632/45917 [8:20:34<4:25:53, 1.12s/it][2025-04-24 22:52:00] (step=0077550) Train Loss: 6.3118, Train Steps/Sec: 0.90 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 31657/45917 [8:21:02<4:21:25, 1.10s/it][2025-04-24 22:52:27] (step=0077575) Train Loss: 6.3296, Train Steps/Sec: 0.90 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 31682/45917 [8:21:29<4:20:37, 1.10s/it][2025-04-24 22:52:55] (step=0077600) Train Loss: 6.2579, Train Steps/Sec: 0.90 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 31707/45917 [8:21:57<4:26:02, 1.12s/it][2025-04-24 22:53:23] (step=0077625) Train Loss: 6.2781, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████ | 31732/45917 [8:22:25<4:25:23, 1.12s/it][2025-04-24 22:53:51] (step=0077650) Train Loss: 6.2629, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 31757/45917 [8:22:53<4:18:44, 1.10s/it][2025-04-24 22:54:18] (step=0077675) Train Loss: 6.2349, Train Steps/Sec: 0.91 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 31782/45917 [8:23:20<4:17:13, 1.09s/it][2025-04-24 22:54:46] (step=0077700) Train Loss: 6.3132, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 31807/45917 [8:23:48<4:24:18, 1.12s/it][2025-04-24 22:55:14] (step=0077725) Train Loss: 6.3177, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 31832/45917 [8:24:16<4:20:49, 1.11s/it][2025-04-24 22:55:42] (step=0077750) Train Loss: 6.2808, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 31857/45917 [8:24:44<4:18:51, 1.10s/it][2025-04-24 22:56:09] (step=0077775) Train Loss: 6.3115, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 31882/45917 [8:25:11<4:15:51, 1.09s/it][2025-04-24 22:56:37] (step=0077800) Train Loss: 6.3009, Train Steps/Sec: 0.90 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 31907/45917 [8:25:39<4:23:28, 1.13s/it][2025-04-24 22:57:05] (step=0077825) Train Loss: 6.2597, Train Steps/Sec: 0.90 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 31932/45917 [8:26:07<4:18:39, 1.11s/it][2025-04-24 22:57:33] (step=0077850) Train Loss: 6.2705, Train Steps/Sec: 0.90 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 31957/45917 [8:26:35<4:16:39, 1.10s/it][2025-04-24 22:58:00] (step=0077875) Train Loss: 6.2567, Train Steps/Sec: 0.90 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 31982/45917 [8:27:02<4:13:04, 1.09s/it][2025-04-24 22:58:28] (step=0077900) Train Loss: 6.3009, Train Steps/Sec: 0.90 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 32007/45917 [8:27:30<4:20:19, 1.12s/it][2025-04-24 22:58:56] (step=0077925) Train Loss: 6.3172, Train Steps/Sec: 0.91 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 32032/45917 [8:27:58<4:14:56, 1.10s/it][2025-04-24 22:59:23] (step=0077950) Train Loss: 6.2977, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 32057/45917 [8:28:25<4:16:23, 1.11s/it][2025-04-24 22:59:51] (step=0077975) Train Loss: 6.3201, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 32082/45917 [8:28:53<4:12:35, 1.10s/it][2025-04-24 23:00:19] (step=0078000) Train Loss: 6.3455, Train Steps/Sec: 0.90 +[2025-04-24 23:00:19] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 23:00:19] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 23:00:19] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 59.16s/it] +[2025-04-24 23:05:04] Finish Eval in 78000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 58.75s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 23:05:25] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0078000.pt +[2025-04-24 23:05:27] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0076000.pt + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 32107/45917 [8:34:29<4:22:28, 1.14s/it][2025-04-24 23:05:55] (step=0078025) Train Loss: 6.2432, Train Steps/Sec: 0.07 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 32132/45917 [8:34:57<4:16:51, 1.12s/it][2025-04-24 23:06:23] (step=0078050) Train Loss: 6.2832, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 32157/45917 [8:35:25<4:14:28, 1.11s/it][2025-04-24 23:06:51] (step=0078075) Train Loss: 6.3157, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32182/45917 [8:35:53<4:12:02, 1.10s/it][2025-04-24 23:07:19] (step=0078100) Train Loss: 6.2525, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32207/45917 [8:36:20<4:20:07, 1.14s/it][2025-04-24 23:07:46] (step=0078125) Train Loss: 6.3142, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 32232/45917 [8:36:48<4:12:16, 1.11s/it][2025-04-24 23:08:14] (step=0078150) Train Loss: 6.2946, Train Steps/Sec: 0.91 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32257/45917 [8:37:16<4:08:51, 1.09s/it][2025-04-24 23:08:41] (step=0078175) Train Loss: 6.2980, Train Steps/Sec: 0.91 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32282/45917 [8:37:43<4:07:19, 1.09s/it][2025-04-24 23:09:09] (step=0078200) Train Loss: 6.3420, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 32307/45917 [8:38:11<4:18:42, 1.14s/it][2025-04-24 23:09:37] (step=0078225) Train Loss: 6.3538, Train Steps/Sec: 0.89 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 32332/45917 [8:38:39<4:10:11, 1.11s/it][2025-04-24 23:10:05] (step=0078250) Train Loss: 6.3438, Train Steps/Sec: 0.90 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 32357/45917 [8:39:07<4:09:53, 1.11s/it][2025-04-24 23:10:33] (step=0078275) Train Loss: 6.3317, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 32382/45917 [8:39:35<4:07:35, 1.10s/it][2025-04-24 23:11:01] (step=0078300) Train Loss: 6.3116, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 32407/45917 [8:40:02<4:13:57, 1.13s/it][2025-04-24 23:11:28] (step=0078325) Train Loss: 6.2943, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 32432/45917 [8:40:30<4:09:33, 1.11s/it][2025-04-24 23:11:56] (step=0078350) Train Loss: 6.2936, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 32457/45917 [8:40:58<4:08:45, 1.11s/it][2025-04-24 23:12:24] (step=0078375) Train Loss: 6.3334, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 32482/45917 [8:41:25<4:06:23, 1.10s/it][2025-04-24 23:12:51] (step=0078400) Train Loss: 6.3046, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32507/45917 [8:41:53<4:11:38, 1.13s/it][2025-04-24 23:13:19] (step=0078425) Train Loss: 6.2960, Train Steps/Sec: 0.91 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32532/45917 [8:42:21<4:04:25, 1.10s/it][2025-04-24 23:13:47] (step=0078450) Train Loss: 6.2917, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 32557/45917 [8:42:49<4:03:58, 1.10s/it][2025-04-24 23:14:14] (step=0078475) Train Loss: 6.3275, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32582/45917 [8:43:16<4:04:05, 1.10s/it][2025-04-24 23:14:42] (step=0078500) Train Loss: 6.3280, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32607/45917 [8:43:44<4:11:34, 1.13s/it][2025-04-24 23:15:10] (step=0078525) Train Loss: 6.3558, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 32632/45917 [8:44:12<4:04:13, 1.10s/it][2025-04-24 23:15:37] (step=0078550) Train Loss: 6.2865, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 32657/45917 [8:44:39<4:02:47, 1.10s/it][2025-04-24 23:16:05] (step=0078575) Train Loss: 6.2856, Train Steps/Sec: 0.90 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 32682/45917 [8:45:07<4:03:29, 1.10s/it][2025-04-24 23:16:33] (step=0078600) Train Loss: 6.3417, Train Steps/Sec: 0.90 + 71%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 32707/45917 [8:45:35<4:06:35, 1.12s/it][2025-04-24 23:17:01] (step=0078625) Train Loss: 6.3110, Train Steps/Sec: 0.90 + 71%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 32732/45917 [8:46:03<4:03:23, 1.11s/it][2025-04-24 23:17:28] (step=0078650) Train Loss: 6.2943, Train Steps/Sec: 0.90 + 71%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 32757/45917 [8:46:30<4:00:06, 1.09s/it][2025-04-24 23:17:56] (step=0078675) Train Loss: 6.3399, Train Steps/Sec: 0.91 + 71%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 32782/45917 [8:46:58<4:02:20, 1.11s/it][2025-04-24 23:18:24] (step=0078700) Train Loss: 6.3091, Train Steps/Sec: 0.89 + 71%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 32807/45917 [8:47:26<4:06:00, 1.13s/it][2025-04-24 23:18:52] (step=0078725) Train Loss: 6.3275, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32832/45917 [8:47:54<4:01:46, 1.11s/it][2025-04-24 23:19:19] (step=0078750) Train Loss: 6.3043, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32857/45917 [8:48:21<3:59:21, 1.10s/it][2025-04-24 23:19:47] (step=0078775) Train Loss: 6.3055, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 32882/45917 [8:48:49<4:00:03, 1.10s/it][2025-04-24 23:20:15] (step=0078800) Train Loss: 6.2898, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 32907/45917 [8:49:17<4:04:34, 1.13s/it][2025-04-24 23:20:43] (step=0078825) Train Loss: 6.3370, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32932/45917 [8:49:45<4:01:26, 1.12s/it][2025-04-24 23:21:10] (step=0078850) Train Loss: 6.3222, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 32957/45917 [8:50:12<3:58:18, 1.10s/it][2025-04-24 23:21:38] (step=0078875) Train Loss: 6.2976, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 32982/45917 [8:50:40<3:57:34, 1.10s/it][2025-04-24 23:22:06] (step=0078900) Train Loss: 6.3148, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 33007/45917 [8:51:08<4:03:36, 1.13s/it][2025-04-24 23:22:34] (step=0078925) Train Loss: 6.3127, Train Steps/Sec: 0.90 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 33032/45917 [8:51:36<3:58:31, 1.11s/it][2025-04-24 23:23:01] (step=0078950) Train Loss: 6.3443, Train Steps/Sec: 0.90 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 33057/45917 [8:52:03<3:54:50, 1.10s/it][2025-04-24 23:23:29] (step=0078975) Train Loss: 6.2860, Train Steps/Sec: 0.91 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 33082/45917 [8:52:31<3:52:58, 1.09s/it][2025-04-24 23:23:57] (step=0079000) Train Loss: 6.2488, Train Steps/Sec: 0.90 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 33107/45917 [8:52:58<3:58:42, 1.12s/it][2025-04-24 23:24:24] (step=0079025) Train Loss: 6.2944, Train Steps/Sec: 0.91 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 33132/45917 [8:53:26<3:54:29, 1.10s/it][2025-04-24 23:24:52] (step=0079050) Train Loss: 6.2851, Train Steps/Sec: 0.91 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 33157/45917 [8:53:54<3:56:22, 1.11s/it][2025-04-24 23:25:20] (step=0079075) Train Loss: 6.3149, Train Steps/Sec: 0.90 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 33182/45917 [8:54:21<3:52:00, 1.09s/it][2025-04-24 23:25:47] (step=0079100) Train Loss: 6.3478, Train Steps/Sec: 0.90 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33207/45917 [8:54:49<3:58:09, 1.12s/it][2025-04-24 23:26:15] (step=0079125) Train Loss: 6.2965, Train Steps/Sec: 0.91 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33232/45917 [8:55:17<3:53:28, 1.10s/it][2025-04-24 23:26:43] (step=0079150) Train Loss: 6.3112, Train Steps/Sec: 0.91 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 33257/45917 [8:55:44<3:53:14, 1.11s/it][2025-04-24 23:27:10] (step=0079175) Train Loss: 6.3248, Train Steps/Sec: 0.90 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 33282/45917 [8:56:12<3:51:34, 1.10s/it][2025-04-24 23:27:38] (step=0079200) Train Loss: 6.2991, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 33307/45917 [8:56:40<3:57:40, 1.13s/it][2025-04-24 23:28:06] (step=0079225) Train Loss: 6.3004, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 33332/45917 [8:57:08<3:53:19, 1.11s/it][2025-04-24 23:28:33] (step=0079250) Train Loss: 6.2923, Train Steps/Sec: 0.90 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 33357/45917 [8:57:35<3:49:30, 1.10s/it][2025-04-24 23:29:01] (step=0079275) Train Loss: 6.3150, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 33382/45917 [8:58:03<3:48:43, 1.09s/it][2025-04-24 23:29:29] (step=0079300) Train Loss: 6.2325, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 33407/45917 [8:58:31<3:58:04, 1.14s/it][2025-04-24 23:29:57] (step=0079325) Train Loss: 6.2862, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 33432/45917 [8:58:59<3:51:11, 1.11s/it][2025-04-24 23:30:24] (step=0079350) Train Loss: 6.2900, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 33457/45917 [8:59:26<3:48:40, 1.10s/it][2025-04-24 23:30:52] (step=0079375) Train Loss: 6.3329, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 33482/45917 [8:59:54<3:48:16, 1.10s/it][2025-04-24 23:31:20] (step=0079400) Train Loss: 6.2989, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 33507/45917 [9:00:27<4:16:53, 1.24s/it][2025-04-24 23:31:53] (step=0079425) Train Loss: 6.3405, Train Steps/Sec: 0.77 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33532/45917 [9:00:55<3:52:05, 1.12s/it][2025-04-24 23:32:20] (step=0079450) Train Loss: 6.3465, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33557/45917 [9:01:22<3:47:33, 1.10s/it][2025-04-24 23:32:48] (step=0079475) Train Loss: 6.3175, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 33582/45917 [9:01:50<3:43:22, 1.09s/it][2025-04-24 23:33:16] (step=0079500) Train Loss: 6.2847, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 33607/45917 [9:02:18<3:50:25, 1.12s/it][2025-04-24 23:33:43] (step=0079525) Train Loss: 6.2775, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 33632/45917 [9:02:45<3:48:00, 1.11s/it][2025-04-24 23:34:11] (step=0079550) Train Loss: 6.2837, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 33657/45917 [9:03:13<3:44:30, 1.10s/it][2025-04-24 23:34:39] (step=0079575) Train Loss: 6.3318, Train Steps/Sec: 0.90 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 33682/45917 [9:03:41<3:45:33, 1.11s/it][2025-04-24 23:35:07] (step=0079600) Train Loss: 6.2265, Train Steps/Sec: 0.90 + 73%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 33707/45917 [9:04:13<3:51:33, 1.14s/it][2025-04-24 23:35:39] (step=0079625) Train Loss: 6.3514, Train Steps/Sec: 0.77 + 73%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 33732/45917 [9:04:41<3:44:35, 1.11s/it][2025-04-24 23:36:07] (step=0079650) Train Loss: 6.2594, Train Steps/Sec: 0.90 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 33757/45917 [9:05:09<3:43:21, 1.10s/it][2025-04-24 23:36:35] (step=0079675) Train Loss: 6.2761, Train Steps/Sec: 0.90 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 33782/45917 [9:05:37<3:41:32, 1.10s/it][2025-04-24 23:37:02] (step=0079700) Train Loss: 6.3271, Train Steps/Sec: 0.90 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 33807/45917 [9:06:09<3:48:48, 1.13s/it][2025-04-24 23:37:35] (step=0079725) Train Loss: 6.3209, Train Steps/Sec: 0.76 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 33832/45917 [9:06:37<3:44:15, 1.11s/it][2025-04-24 23:38:03] (step=0079750) Train Loss: 6.2610, Train Steps/Sec: 0.90 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 33857/45917 [9:07:05<3:41:11, 1.10s/it][2025-04-24 23:38:31] (step=0079775) Train Loss: 6.2846, Train Steps/Sec: 0.90 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33882/45917 [9:07:33<3:39:55, 1.10s/it][2025-04-24 23:38:58] (step=0079800) Train Loss: 6.2309, Train Steps/Sec: 0.90 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 33907/45917 [9:08:00<3:43:43, 1.12s/it][2025-04-24 23:39:26] (step=0079825) Train Loss: 6.2704, Train Steps/Sec: 0.91 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 33932/45917 [9:08:33<3:46:13, 1.13s/it][2025-04-24 23:39:59] (step=0079850) Train Loss: 6.2802, Train Steps/Sec: 0.76 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 33957/45917 [9:09:06<3:38:43, 1.10s/it][2025-04-24 23:40:32] (step=0079875) Train Loss: 6.2863, Train Steps/Sec: 0.76 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 33982/45917 [9:09:39<3:40:06, 1.11s/it][2025-04-24 23:41:05] (step=0079900) Train Loss: 6.3426, Train Steps/Sec: 0.76 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 34007/45917 [9:10:06<3:43:58, 1.13s/it][2025-04-24 23:41:32] (step=0079925) Train Loss: 6.2735, Train Steps/Sec: 0.90 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 34032/45917 [9:10:34<3:38:30, 1.10s/it][2025-04-24 23:42:00] (step=0079950) Train Loss: 6.2970, Train Steps/Sec: 0.90 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 34057/45917 [9:11:02<3:35:39, 1.09s/it][2025-04-24 23:42:28] (step=0079975) Train Loss: 6.2832, Train Steps/Sec: 0.90 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34082/45917 [9:11:30<3:37:12, 1.10s/it][2025-04-24 23:42:55] (step=0080000) Train Loss: 6.3344, Train Steps/Sec: 0.90 +[2025-04-24 23:42:56] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-24 23:42:56] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-24 23:42:56] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.63s/it] +[2025-04-24 23:47:40] Finish Eval in 80000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.25s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 23:48:01] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0080000.pt +[2025-04-24 23:48:03] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0078000.pt + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34107/45917 [9:17:06<3:44:29, 1.14s/it][2025-04-24 23:48:31] (step=0080025) Train Loss: 6.2396, Train Steps/Sec: 0.07 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 34132/45917 [9:17:33<3:37:07, 1.11s/it][2025-04-24 23:48:59] (step=0080050) Train Loss: 6.2821, Train Steps/Sec: 0.90 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 34157/45917 [9:18:01<3:35:52, 1.10s/it][2025-04-24 23:49:27] (step=0080075) Train Loss: 6.3444, Train Steps/Sec: 0.90 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 34182/45917 [9:18:29<3:35:07, 1.10s/it][2025-04-24 23:49:55] (step=0080100) Train Loss: 6.3113, Train Steps/Sec: 0.89 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 34207/45917 [9:18:57<3:39:21, 1.12s/it][2025-04-24 23:50:22] (step=0080125) Train Loss: 6.2987, Train Steps/Sec: 0.90 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 34232/45917 [9:19:24<3:35:36, 1.11s/it][2025-04-24 23:50:50] (step=0080150) Train Loss: 6.3200, Train Steps/Sec: 0.90 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 34257/45917 [9:19:52<3:34:16, 1.10s/it][2025-04-24 23:51:18] (step=0080175) Train Loss: 6.3247, Train Steps/Sec: 0.90 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 34282/45917 [9:20:20<3:33:18, 1.10s/it][2025-04-24 23:51:46] (step=0080200) Train Loss: 6.2890, Train Steps/Sec: 0.90 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 34307/45917 [9:20:48<3:36:54, 1.12s/it][2025-04-24 23:52:13] (step=0080225) Train Loss: 6.2853, Train Steps/Sec: 0.90 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 34332/45917 [9:21:15<3:33:09, 1.10s/it][2025-04-24 23:52:41] (step=0080250) Train Loss: 6.2797, Train Steps/Sec: 0.91 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 34357/45917 [9:21:43<3:31:54, 1.10s/it][2025-04-24 23:53:09] (step=0080275) Train Loss: 6.3013, Train Steps/Sec: 0.90 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 34382/45917 [9:22:11<3:30:49, 1.10s/it][2025-04-24 23:53:36] (step=0080300) Train Loss: 6.2952, Train Steps/Sec: 0.90 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34407/45917 [9:22:38<3:35:58, 1.13s/it][2025-04-24 23:54:04] (step=0080325) Train Loss: 6.3080, Train Steps/Sec: 0.90 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34432/45917 [9:23:06<3:33:50, 1.12s/it][2025-04-24 23:54:32] (step=0080350) Train Loss: 6.3039, Train Steps/Sec: 0.90 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 34457/45917 [9:23:34<3:29:32, 1.10s/it][2025-04-24 23:55:00] (step=0080375) Train Loss: 6.3413, Train Steps/Sec: 0.90 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 34482/45917 [9:24:02<3:28:13, 1.09s/it][2025-04-24 23:55:27] (step=0080400) Train Loss: 6.3499, Train Steps/Sec: 0.90 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 34507/45917 [9:24:29<3:33:27, 1.12s/it][2025-04-24 23:55:55] (step=0080425) Train Loss: 6.3143, Train Steps/Sec: 0.91 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 34532/45917 [9:24:57<3:30:07, 1.11s/it][2025-04-24 23:56:23] (step=0080450) Train Loss: 6.3505, Train Steps/Sec: 0.90 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 34557/45917 [9:25:25<3:28:44, 1.10s/it][2025-04-24 23:56:51] (step=0080475) Train Loss: 6.3251, Train Steps/Sec: 0.90 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 34582/45917 [9:25:52<3:25:19, 1.09s/it][2025-04-24 23:57:18] (step=0080500) Train Loss: 6.2631, Train Steps/Sec: 0.90 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 34607/45917 [9:26:20<3:30:46, 1.12s/it][2025-04-24 23:57:46] (step=0080525) Train Loss: 6.2819, Train Steps/Sec: 0.90 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 34632/45917 [9:26:48<3:28:20, 1.11s/it][2025-04-24 23:58:14] (step=0080550) Train Loss: 6.2579, Train Steps/Sec: 0.90 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 34657/45917 [9:27:16<3:27:06, 1.10s/it][2025-04-24 23:58:41] (step=0080575) Train Loss: 6.3120, Train Steps/Sec: 0.90 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 34682/45917 [9:27:43<3:27:00, 1.11s/it][2025-04-24 23:59:09] (step=0080600) Train Loss: 6.2842, Train Steps/Sec: 0.90 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 34707/45917 [9:28:11<3:30:56, 1.13s/it][2025-04-24 23:59:37] (step=0080625) Train Loss: 6.2931, Train Steps/Sec: 0.90 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34732/45917 [9:28:39<3:27:02, 1.11s/it][2025-04-25 00:00:05] (step=0080650) Train Loss: 6.2838, Train Steps/Sec: 0.90 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34757/45917 [9:29:07<3:22:55, 1.09s/it][2025-04-25 00:00:32] (step=0080675) Train Loss: 6.2908, Train Steps/Sec: 0.90 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 34782/45917 [9:29:34<3:25:15, 1.11s/it][2025-04-25 00:01:00] (step=0080700) Train Loss: 6.2675, Train Steps/Sec: 0.89 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 34807/45917 [9:30:02<3:29:03, 1.13s/it][2025-04-25 00:01:28] (step=0080725) Train Loss: 6.3007, Train Steps/Sec: 0.90 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 34832/45917 [9:30:30<3:25:14, 1.11s/it][2025-04-25 00:01:56] (step=0080750) Train Loss: 6.3461, Train Steps/Sec: 0.91 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 34857/45917 [9:30:58<3:22:30, 1.10s/it][2025-04-25 00:02:23] (step=0080775) Train Loss: 6.2940, Train Steps/Sec: 0.90 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 34882/45917 [9:31:25<3:21:18, 1.09s/it][2025-04-25 00:02:51] (step=0080800) Train Loss: 6.3360, Train Steps/Sec: 0.90 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 34907/45917 [9:31:53<3:26:45, 1.13s/it][2025-04-25 00:03:19] (step=0080825) Train Loss: 6.3183, Train Steps/Sec: 0.91 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 34932/45917 [9:32:21<3:23:25, 1.11s/it][2025-04-25 00:03:46] (step=0080850) Train Loss: 6.2951, Train Steps/Sec: 0.90 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 34957/45917 [9:32:48<3:23:06, 1.11s/it][2025-04-25 00:04:14] (step=0080875) Train Loss: 6.2900, Train Steps/Sec: 0.90 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 34982/45917 [9:33:16<3:18:38, 1.09s/it][2025-04-25 00:04:42] (step=0080900) Train Loss: 6.3014, Train Steps/Sec: 0.90 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35007/45917 [9:33:44<3:24:41, 1.13s/it][2025-04-25 00:05:09] (step=0080925) Train Loss: 6.3155, Train Steps/Sec: 0.91 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 35032/45917 [9:34:11<3:20:55, 1.11s/it][2025-04-25 00:05:37] (step=0080950) Train Loss: 6.2947, Train Steps/Sec: 0.90 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 35057/45917 [9:34:39<3:20:17, 1.11s/it][2025-04-25 00:06:05] (step=0080975) Train Loss: 6.2528, Train Steps/Sec: 0.90 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35082/45917 [9:35:07<3:18:37, 1.10s/it][2025-04-25 00:06:33] (step=0081000) Train Loss: 6.2862, Train Steps/Sec: 0.90 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35107/45917 [9:35:34<3:21:26, 1.12s/it][2025-04-25 00:07:00] (step=0081025) Train Loss: 6.2918, Train Steps/Sec: 0.91 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35132/45917 [9:36:02<3:20:12, 1.11s/it][2025-04-25 00:07:28] (step=0081050) Train Loss: 6.3145, Train Steps/Sec: 0.90 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35157/45917 [9:36:30<3:17:03, 1.10s/it][2025-04-25 00:07:56] (step=0081075) Train Loss: 6.2846, Train Steps/Sec: 0.90 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35182/45917 [9:36:58<3:15:33, 1.09s/it][2025-04-25 00:08:24] (step=0081100) Train Loss: 6.3027, Train Steps/Sec: 0.90 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35207/45917 [9:37:26<3:21:20, 1.13s/it][2025-04-25 00:08:51] (step=0081125) Train Loss: 6.3519, Train Steps/Sec: 0.90 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35232/45917 [9:37:53<3:16:40, 1.10s/it][2025-04-25 00:09:19] (step=0081150) Train Loss: 6.3011, Train Steps/Sec: 0.90 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35257/45917 [9:38:21<3:16:56, 1.11s/it][2025-04-25 00:09:47] (step=0081175) Train Loss: 6.3314, Train Steps/Sec: 0.90 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35282/45917 [9:38:49<3:14:51, 1.10s/it][2025-04-25 00:10:15] (step=0081200) Train Loss: 6.2779, Train Steps/Sec: 0.90 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35307/45917 [9:39:17<3:19:40, 1.13s/it][2025-04-25 00:10:42] (step=0081225) Train Loss: 6.3105, Train Steps/Sec: 0.90 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35332/45917 [9:39:44<3:15:23, 1.11s/it][2025-04-25 00:11:10] (step=0081250) Train Loss: 6.2812, Train Steps/Sec: 0.90 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 35357/45917 [9:40:12<3:14:56, 1.11s/it][2025-04-25 00:11:38] (step=0081275) Train Loss: 6.3224, Train Steps/Sec: 0.90 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 35382/45917 [9:40:40<3:13:15, 1.10s/it][2025-04-25 00:12:06] (step=0081300) Train Loss: 6.3541, Train Steps/Sec: 0.90 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35407/45917 [9:41:07<3:16:32, 1.12s/it][2025-04-25 00:12:33] (step=0081325) Train Loss: 6.2901, Train Steps/Sec: 0.91 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35432/45917 [9:41:35<3:13:02, 1.10s/it][2025-04-25 00:13:01] (step=0081350) Train Loss: 6.3214, Train Steps/Sec: 0.91 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35457/45917 [9:42:03<3:11:34, 1.10s/it][2025-04-25 00:13:28] (step=0081375) Train Loss: 6.2894, Train Steps/Sec: 0.91 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35482/45917 [9:42:30<3:10:50, 1.10s/it][2025-04-25 00:13:56] (step=0081400) Train Loss: 6.2600, Train Steps/Sec: 0.90 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35507/45917 [9:42:58<3:15:06, 1.12s/it][2025-04-25 00:14:24] (step=0081425) Train Loss: 6.2929, Train Steps/Sec: 0.91 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35532/45917 [9:43:26<3:10:35, 1.10s/it][2025-04-25 00:14:51] (step=0081450) Train Loss: 6.3048, Train Steps/Sec: 0.91 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35557/45917 [9:43:53<3:10:13, 1.10s/it][2025-04-25 00:15:19] (step=0081475) Train Loss: 6.3243, Train Steps/Sec: 0.90 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35582/45917 [9:44:21<3:09:48, 1.10s/it][2025-04-25 00:15:47] (step=0081500) Train Loss: 6.3267, Train Steps/Sec: 0.90 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35607/45917 [9:44:49<3:13:37, 1.13s/it][2025-04-25 00:16:14] (step=0081525) Train Loss: 6.3252, Train Steps/Sec: 0.91 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35632/45917 [9:45:16<3:11:04, 1.11s/it][2025-04-25 00:16:42] (step=0081550) Train Loss: 6.3091, Train Steps/Sec: 0.90 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35657/45917 [9:45:44<3:07:16, 1.10s/it][2025-04-25 00:17:10] (step=0081575) Train Loss: 6.2037, Train Steps/Sec: 0.91 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 35682/45917 [9:46:17<3:08:16, 1.10s/it][2025-04-25 00:17:43] (step=0081600) Train Loss: 6.3330, Train Steps/Sec: 0.75 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 35707/45917 [9:46:45<3:13:58, 1.14s/it][2025-04-25 00:18:11] (step=0081625) Train Loss: 6.2705, Train Steps/Sec: 0.90 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35732/45917 [9:47:12<3:06:48, 1.10s/it][2025-04-25 00:18:38] (step=0081650) Train Loss: 6.3199, Train Steps/Sec: 0.91 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35757/45917 [9:47:40<3:06:58, 1.10s/it][2025-04-25 00:19:06] (step=0081675) Train Loss: 6.2925, Train Steps/Sec: 0.91 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35782/45917 [9:48:08<3:04:57, 1.10s/it][2025-04-25 00:19:34] (step=0081700) Train Loss: 6.3071, Train Steps/Sec: 0.90 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35807/45917 [9:48:35<3:09:14, 1.12s/it][2025-04-25 00:20:01] (step=0081725) Train Loss: 6.2793, Train Steps/Sec: 0.90 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35832/45917 [9:49:03<3:05:06, 1.10s/it][2025-04-25 00:20:29] (step=0081750) Train Loss: 6.3074, Train Steps/Sec: 0.91 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35857/45917 [9:49:31<3:06:12, 1.11s/it][2025-04-25 00:20:56] (step=0081775) Train Loss: 6.2620, Train Steps/Sec: 0.90 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35882/45917 [9:49:58<3:02:20, 1.09s/it][2025-04-25 00:21:24] (step=0081800) Train Loss: 6.3373, Train Steps/Sec: 0.90 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35907/45917 [9:50:26<3:06:57, 1.12s/it][2025-04-25 00:21:52] (step=0081825) Train Loss: 6.3354, Train Steps/Sec: 0.90 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35932/45917 [9:50:54<3:05:04, 1.11s/it][2025-04-25 00:22:20] (step=0081850) Train Loss: 6.3243, Train Steps/Sec: 0.90 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35957/45917 [9:51:22<3:04:16, 1.11s/it][2025-04-25 00:22:47] (step=0081875) Train Loss: 6.2798, Train Steps/Sec: 0.90 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35982/45917 [9:51:49<3:00:54, 1.09s/it][2025-04-25 00:23:15] (step=0081900) Train Loss: 6.2984, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36007/45917 [9:52:17<3:07:33, 1.14s/it][2025-04-25 00:23:43] (step=0081925) Train Loss: 6.2479, Train Steps/Sec: 0.90 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36032/45917 [9:52:45<3:05:52, 1.13s/it][2025-04-25 00:24:10] (step=0081950) Train Loss: 6.3009, Train Steps/Sec: 0.90 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36057/45917 [9:53:12<3:01:07, 1.10s/it][2025-04-25 00:24:38] (step=0081975) Train Loss: 6.2692, Train Steps/Sec: 0.90 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36082/45917 [9:53:40<2:59:45, 1.10s/it][2025-04-25 00:25:06] (step=0082000) Train Loss: 6.2803, Train Steps/Sec: 0.90 +[2025-04-25 00:25:06] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-25 00:25:06] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-25 00:25:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 59.01s/it] +[2025-04-25 00:29:51] Finish Eval in 82000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.57s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 00:30:13] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0082000.pt +[2025-04-25 00:30:15] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0080000.pt + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36107/45917 [9:59:17<3:07:06, 1.14s/it][2025-04-25 00:30:43] (step=0082025) Train Loss: 6.2842, Train Steps/Sec: 0.07 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36132/45917 [9:59:45<3:01:39, 1.11s/it][2025-04-25 00:31:10] (step=0082050) Train Loss: 6.2654, Train Steps/Sec: 0.90 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36157/45917 [10:00:12<2:57:26, 1.09s/it][2025-04-25 00:31:38] (step=0082075) Train Loss: 6.2760, Train Steps/Sec: 0.91 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36182/45917 [10:00:40<2:59:10, 1.10s/it][2025-04-25 00:32:06] (step=0082100) Train Loss: 6.2827, Train Steps/Sec: 0.90 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36207/45917 [10:01:08<3:03:32, 1.13s/it][2025-04-25 00:32:34] (step=0082125) Train Loss: 6.3121, Train Steps/Sec: 0.90 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36232/45917 [10:01:35<2:59:28, 1.11s/it][2025-04-25 00:33:01] (step=0082150) Train Loss: 6.2911, Train Steps/Sec: 0.90 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36257/45917 [10:02:03<2:57:06, 1.10s/it][2025-04-25 00:33:29] (step=0082175) Train Loss: 6.2983, Train Steps/Sec: 0.91 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36282/45917 [10:02:31<2:55:59, 1.10s/it][2025-04-25 00:33:57] (step=0082200) Train Loss: 6.2521, Train Steps/Sec: 0.90 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36307/45917 [10:02:59<2:59:31, 1.12s/it][2025-04-25 00:34:24] (step=0082225) Train Loss: 6.3287, Train Steps/Sec: 0.90 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36332/45917 [10:03:26<2:57:29, 1.11s/it][2025-04-25 00:34:52] (step=0082250) Train Loss: 6.3299, Train Steps/Sec: 0.90 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36357/45917 [10:03:54<2:55:04, 1.10s/it][2025-04-25 00:35:20] (step=0082275) Train Loss: 6.2717, Train Steps/Sec: 0.91 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36382/45917 [10:04:21<2:53:07, 1.09s/it][2025-04-25 00:35:47] (step=0082300) Train Loss: 6.3402, Train Steps/Sec: 0.90 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36407/45917 [10:04:49<2:58:57, 1.13s/it][2025-04-25 00:36:15] (step=0082325) Train Loss: 6.2813, Train Steps/Sec: 0.90 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36432/45917 [10:05:17<2:55:41, 1.11s/it][2025-04-25 00:36:42] (step=0082350) Train Loss: 6.3512, Train Steps/Sec: 0.91 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36457/45917 [10:05:44<2:53:53, 1.10s/it][2025-04-25 00:37:10] (step=0082375) Train Loss: 6.3024, Train Steps/Sec: 0.90 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36482/45917 [10:06:12<2:51:28, 1.09s/it][2025-04-25 00:37:38] (step=0082400) Train Loss: 6.2664, Train Steps/Sec: 0.90 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36507/45917 [10:06:40<2:55:35, 1.12s/it][2025-04-25 00:38:05] (step=0082425) Train Loss: 6.2551, Train Steps/Sec: 0.91 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36532/45917 [10:07:07<2:53:12, 1.11s/it][2025-04-25 00:38:33] (step=0082450) Train Loss: 6.3076, Train Steps/Sec: 0.90 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36557/45917 [10:07:35<2:52:32, 1.11s/it][2025-04-25 00:39:01] (step=0082475) Train Loss: 6.2789, Train Steps/Sec: 0.90 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36582/45917 [10:08:03<2:49:43, 1.09s/it][2025-04-25 00:39:29] (step=0082500) Train Loss: 6.3366, Train Steps/Sec: 0.90 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36607/45917 [10:08:31<2:54:20, 1.12s/it][2025-04-25 00:39:56] (step=0082525) Train Loss: 6.2961, Train Steps/Sec: 0.91 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36632/45917 [10:08:58<2:50:42, 1.10s/it][2025-04-25 00:40:24] (step=0082550) Train Loss: 6.2779, Train Steps/Sec: 0.90 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36657/45917 [10:09:26<2:50:43, 1.11s/it][2025-04-25 00:40:52] (step=0082575) Train Loss: 6.2716, Train Steps/Sec: 0.90 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36682/45917 [10:09:54<2:49:56, 1.10s/it][2025-04-25 00:41:20] (step=0082600) Train Loss: 6.3244, Train Steps/Sec: 0.89 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36707/45917 [10:10:22<2:52:43, 1.13s/it][2025-04-25 00:41:47] (step=0082625) Train Loss: 6.3539, Train Steps/Sec: 0.90 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36732/45917 [10:10:49<2:49:20, 1.11s/it][2025-04-25 00:42:15] (step=0082650) Train Loss: 6.3210, Train Steps/Sec: 0.90 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36757/45917 [10:11:17<2:48:26, 1.10s/it][2025-04-25 00:42:43] (step=0082675) Train Loss: 6.2843, Train Steps/Sec: 0.90 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36782/45917 [10:11:45<2:46:48, 1.10s/it][2025-04-25 00:43:11] (step=0082700) Train Loss: 6.2763, Train Steps/Sec: 0.90 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36807/45917 [10:12:12<2:49:14, 1.11s/it][2025-04-25 00:43:38] (step=0082725) Train Loss: 6.2562, Train Steps/Sec: 0.91 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36832/45917 [10:12:40<2:46:46, 1.10s/it][2025-04-25 00:44:06] (step=0082750) Train Loss: 6.3079, Train Steps/Sec: 0.90 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36857/45917 [10:13:08<2:47:01, 1.11s/it][2025-04-25 00:44:34] (step=0082775) Train Loss: 6.3034, Train Steps/Sec: 0.90 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36882/45917 [10:13:35<2:44:23, 1.09s/it][2025-04-25 00:45:01] (step=0082800) Train Loss: 6.2875, Train Steps/Sec: 0.90 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36907/45917 [10:14:03<2:50:07, 1.13s/it][2025-04-25 00:45:29] (step=0082825) Train Loss: 6.3148, Train Steps/Sec: 0.90 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36932/45917 [10:14:31<2:44:53, 1.10s/it][2025-04-25 00:45:57] (step=0082850) Train Loss: 6.2617, Train Steps/Sec: 0.91 + 80%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36957/45917 [10:14:59<2:46:33, 1.12s/it][2025-04-25 00:46:24] (step=0082875) Train Loss: 6.2635, Train Steps/Sec: 0.90 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36982/45917 [10:15:26<2:43:22, 1.10s/it][2025-04-25 00:46:52] (step=0082900) Train Loss: 6.3300, Train Steps/Sec: 0.90 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37007/45917 [10:15:54<2:47:25, 1.13s/it][2025-04-25 00:47:20] (step=0082925) Train Loss: 6.2996, Train Steps/Sec: 0.90 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37032/45917 [10:16:22<2:45:45, 1.12s/it][2025-04-25 00:47:48] (step=0082950) Train Loss: 6.3010, Train Steps/Sec: 0.90 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37057/45917 [10:16:50<2:43:24, 1.11s/it][2025-04-25 00:48:15] (step=0082975) Train Loss: 6.2986, Train Steps/Sec: 0.90 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37082/45917 [10:17:17<2:40:49, 1.09s/it][2025-04-25 00:48:43] (step=0083000) Train Loss: 6.2627, Train Steps/Sec: 0.90 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37107/45917 [10:17:45<2:45:10, 1.12s/it][2025-04-25 00:49:11] (step=0083025) Train Loss: 6.2760, Train Steps/Sec: 0.91 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37132/45917 [10:18:13<2:41:57, 1.11s/it][2025-04-25 00:49:39] (step=0083050) Train Loss: 6.2924, Train Steps/Sec: 0.90 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37157/45917 [10:18:40<2:40:28, 1.10s/it][2025-04-25 00:50:06] (step=0083075) Train Loss: 6.2975, Train Steps/Sec: 0.90 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37182/45917 [10:19:08<2:39:06, 1.09s/it][2025-04-25 00:50:34] (step=0083100) Train Loss: 6.3488, Train Steps/Sec: 0.90 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37207/45917 [10:19:36<2:42:53, 1.12s/it][2025-04-25 00:51:02] (step=0083125) Train Loss: 6.3255, Train Steps/Sec: 0.90 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37232/45917 [10:20:04<2:41:33, 1.12s/it][2025-04-25 00:51:29] (step=0083150) Train Loss: 6.3039, Train Steps/Sec: 0.90 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37257/45917 [10:20:31<2:38:45, 1.10s/it][2025-04-25 00:51:57] (step=0083175) Train Loss: 6.2618, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 37282/45917 [10:20:59<2:38:55, 1.10s/it][2025-04-25 00:52:25] (step=0083200) Train Loss: 6.3001, Train Steps/Sec: 0.89 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 37307/45917 [10:21:27<2:41:24, 1.12s/it][2025-04-25 00:52:53] (step=0083225) Train Loss: 6.3143, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37332/45917 [10:21:55<2:38:24, 1.11s/it][2025-04-25 00:53:21] (step=0083250) Train Loss: 6.2765, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37357/45917 [10:22:23<2:37:03, 1.10s/it][2025-04-25 00:53:48] (step=0083275) Train Loss: 6.2852, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37382/45917 [10:22:50<2:37:04, 1.10s/it][2025-04-25 00:54:16] (step=0083300) Train Loss: 6.2621, Train Steps/Sec: 0.90 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37407/45917 [10:23:18<2:40:53, 1.13s/it][2025-04-25 00:54:44] (step=0083325) Train Loss: 6.2605, Train Steps/Sec: 0.90 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37432/45917 [10:23:46<2:37:32, 1.11s/it][2025-04-25 00:55:12] (step=0083350) Train Loss: 6.2825, Train Steps/Sec: 0.90 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37457/45917 [10:24:13<2:35:18, 1.10s/it][2025-04-25 00:55:39] (step=0083375) Train Loss: 6.3069, Train Steps/Sec: 0.90 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37482/45917 [10:24:41<2:33:54, 1.09s/it][2025-04-25 00:56:07] (step=0083400) Train Loss: 6.2700, Train Steps/Sec: 0.90 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37507/45917 [10:25:09<2:37:10, 1.12s/it][2025-04-25 00:56:35] (step=0083425) Train Loss: 6.3034, Train Steps/Sec: 0.91 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37532/45917 [10:25:37<2:34:57, 1.11s/it][2025-04-25 00:57:02] (step=0083450) Train Loss: 6.2897, Train Steps/Sec: 0.90 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37557/45917 [10:26:04<2:33:49, 1.10s/it][2025-04-25 00:57:30] (step=0083475) Train Loss: 6.2665, Train Steps/Sec: 0.90 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37582/45917 [10:26:32<2:31:48, 1.09s/it][2025-04-25 00:57:58] (step=0083500) Train Loss: 6.2871, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 37607/45917 [10:27:00<2:35:45, 1.12s/it][2025-04-25 00:58:26] (step=0083525) Train Loss: 6.2525, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 37632/45917 [10:27:28<2:32:27, 1.10s/it][2025-04-25 00:58:53] (step=0083550) Train Loss: 6.2795, Train Steps/Sec: 0.91 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37657/45917 [10:27:55<2:32:26, 1.11s/it][2025-04-25 00:59:21] (step=0083575) Train Loss: 6.3481, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37682/45917 [10:28:23<2:31:25, 1.10s/it][2025-04-25 00:59:49] (step=0083600) Train Loss: 6.2949, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37707/45917 [10:28:51<2:35:53, 1.14s/it][2025-04-25 01:00:17] (step=0083625) Train Loss: 6.2795, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37732/45917 [10:29:19<2:32:24, 1.12s/it][2025-04-25 01:00:44] (step=0083650) Train Loss: 6.2985, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37757/45917 [10:29:46<2:29:19, 1.10s/it][2025-04-25 01:01:12] (step=0083675) Train Loss: 6.2637, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37782/45917 [10:30:14<2:29:03, 1.10s/it][2025-04-25 01:01:40] (step=0083700) Train Loss: 6.2954, Train Steps/Sec: 0.90 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37807/45917 [10:30:42<2:31:29, 1.12s/it][2025-04-25 01:02:07] (step=0083725) Train Loss: 6.2958, Train Steps/Sec: 0.91 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37832/45917 [10:31:09<2:28:59, 1.11s/it][2025-04-25 01:02:35] (step=0083750) Train Loss: 6.3142, Train Steps/Sec: 0.91 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37857/45917 [10:31:37<2:29:03, 1.11s/it][2025-04-25 01:03:03] (step=0083775) Train Loss: 6.3060, Train Steps/Sec: 0.90 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37882/45917 [10:32:05<2:27:27, 1.10s/it][2025-04-25 01:03:31] (step=0083800) Train Loss: 6.2823, Train Steps/Sec: 0.90 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37907/45917 [10:32:33<2:30:09, 1.12s/it][2025-04-25 01:03:58] (step=0083825) Train Loss: 6.2844, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 37932/45917 [10:33:00<2:27:36, 1.11s/it][2025-04-25 01:04:26] (step=0083850) Train Loss: 6.2959, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 37957/45917 [10:33:28<2:25:57, 1.10s/it][2025-04-25 01:04:54] (step=0083875) Train Loss: 6.2746, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37982/45917 [10:33:56<2:25:58, 1.10s/it][2025-04-25 01:05:21] (step=0083900) Train Loss: 6.2624, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38007/45917 [10:34:23<2:29:38, 1.14s/it][2025-04-25 01:05:49] (step=0083925) Train Loss: 6.3060, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38032/45917 [10:34:51<2:26:32, 1.12s/it][2025-04-25 01:06:17] (step=0083950) Train Loss: 6.3230, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38057/45917 [10:35:19<2:23:38, 1.10s/it][2025-04-25 01:06:45] (step=0083975) Train Loss: 6.3253, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38082/45917 [10:35:47<2:22:40, 1.09s/it][2025-04-25 01:07:13] (step=0084000) Train Loss: 6.3105, Train Steps/Sec: 0.90 +[2025-04-25 01:07:13] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-25 01:07:13] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-25 01:07:13] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.71s/it] +[2025-04-25 01:11:57] Finish Eval in 84000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:55<00:00, 58.24s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 01:12:18] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0084000.pt +[2025-04-25 01:12:20] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0082000.pt + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38107/45917 [10:41:22<2:28:42, 1.14s/it][2025-04-25 01:12:48] (step=0084025) Train Loss: 6.3277, Train Steps/Sec: 0.07 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38132/45917 [10:41:50<2:23:29, 1.11s/it][2025-04-25 01:13:16] (step=0084050) Train Loss: 6.3036, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38157/45917 [10:42:18<2:22:00, 1.10s/it][2025-04-25 01:13:43] (step=0084075) Train Loss: 6.2914, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38182/45917 [10:42:45<2:20:39, 1.09s/it][2025-04-25 01:14:11] (step=0084100) Train Loss: 6.2925, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38207/45917 [10:43:13<2:23:33, 1.12s/it][2025-04-25 01:14:39] (step=0084125) Train Loss: 6.3324, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38232/45917 [10:43:41<2:22:13, 1.11s/it][2025-04-25 01:15:06] (step=0084150) Train Loss: 6.2947, Train Steps/Sec: 0.90 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38257/45917 [10:44:08<2:20:36, 1.10s/it][2025-04-25 01:15:34] (step=0084175) Train Loss: 6.2639, Train Steps/Sec: 0.90 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38282/45917 [10:44:36<2:21:17, 1.11s/it][2025-04-25 01:16:02] (step=0084200) Train Loss: 6.2901, Train Steps/Sec: 0.89 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38307/45917 [10:45:04<2:22:03, 1.12s/it][2025-04-25 01:16:30] (step=0084225) Train Loss: 6.3247, Train Steps/Sec: 0.91 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38332/45917 [10:45:37<2:20:02, 1.11s/it][2025-04-25 01:17:02] (step=0084250) Train Loss: 6.2496, Train Steps/Sec: 0.77 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38357/45917 [10:46:04<2:19:37, 1.11s/it][2025-04-25 01:17:30] (step=0084275) Train Loss: 6.3018, Train Steps/Sec: 0.90 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38382/45917 [10:46:37<2:18:51, 1.11s/it][2025-04-25 01:18:03] (step=0084300) Train Loss: 6.2970, Train Steps/Sec: 0.76 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38407/45917 [10:47:05<2:21:32, 1.13s/it][2025-04-25 01:18:30] (step=0084325) Train Loss: 6.3169, Train Steps/Sec: 0.91 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38432/45917 [10:47:32<2:19:11, 1.12s/it][2025-04-25 01:18:58] (step=0084350) Train Loss: 6.3159, Train Steps/Sec: 0.90 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38457/45917 [10:48:00<2:16:30, 1.10s/it][2025-04-25 01:19:26] (step=0084375) Train Loss: 6.2599, Train Steps/Sec: 0.90 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38482/45917 [10:48:33<5:19:42, 2.58s/it][2025-04-25 01:19:59] (step=0084400) Train Loss: 6.2958, Train Steps/Sec: 0.76 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38507/45917 [10:49:01<2:19:05, 1.13s/it][2025-04-25 01:20:26] (step=0084425) Train Loss: 6.3050, Train Steps/Sec: 0.90 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38532/45917 [10:49:28<2:15:53, 1.10s/it][2025-04-25 01:20:54] (step=0084450) Train Loss: 6.2989, Train Steps/Sec: 0.91 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38557/45917 [10:49:56<2:14:36, 1.10s/it][2025-04-25 01:21:21] (step=0084475) Train Loss: 6.3016, Train Steps/Sec: 0.91 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38582/45917 [10:50:23<2:13:17, 1.09s/it][2025-04-25 01:21:49] (step=0084500) Train Loss: 6.2918, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38607/45917 [10:50:51<2:19:15, 1.14s/it][2025-04-25 01:22:17] (step=0084525) Train Loss: 6.2981, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38632/45917 [10:51:19<2:16:15, 1.12s/it][2025-04-25 01:22:45] (step=0084550) Train Loss: 6.3001, Train Steps/Sec: 0.90 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38657/45917 [10:51:47<2:12:30, 1.10s/it][2025-04-25 01:23:12] (step=0084575) Train Loss: 6.3148, Train Steps/Sec: 0.91 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38682/45917 [10:52:19<2:11:33, 1.09s/it][2025-04-25 01:23:45] (step=0084600) Train Loss: 6.3045, Train Steps/Sec: 0.76 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38707/45917 [10:52:52<5:14:51, 2.62s/it][2025-04-25 01:24:18] (step=0084625) Train Loss: 6.2882, Train Steps/Sec: 0.76 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38732/45917 [10:53:20<2:11:36, 1.10s/it][2025-04-25 01:24:45] (step=0084650) Train Loss: 6.2856, Train Steps/Sec: 0.91 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38757/45917 [10:53:53<2:13:57, 1.12s/it][2025-04-25 01:25:18] (step=0084675) Train Loss: 6.2846, Train Steps/Sec: 0.76 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38782/45917 [10:54:21<2:10:29, 1.10s/it][2025-04-25 01:25:47] (step=0084700) Train Loss: 6.3095, Train Steps/Sec: 0.88 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38807/45917 [10:54:49<2:12:49, 1.12s/it][2025-04-25 01:26:14] (step=0084725) Train Loss: 6.2786, Train Steps/Sec: 0.90 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38832/45917 [10:55:16<2:10:44, 1.11s/it][2025-04-25 01:26:42] (step=0084750) Train Loss: 6.2286, Train Steps/Sec: 0.90 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38857/45917 [10:55:44<2:09:55, 1.10s/it][2025-04-25 01:27:10] (step=0084775) Train Loss: 6.2970, Train Steps/Sec: 0.90 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38882/45917 [10:56:12<2:09:01, 1.10s/it][2025-04-25 01:27:38] (step=0084800) Train Loss: 6.3282, Train Steps/Sec: 0.90 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38907/45917 [10:56:40<2:12:04, 1.13s/it][2025-04-25 01:28:05] (step=0084825) Train Loss: 6.3397, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38932/45917 [10:57:07<2:09:32, 1.11s/it][2025-04-25 01:28:33] (step=0084850) Train Loss: 6.2799, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38957/45917 [10:57:35<2:07:49, 1.10s/it][2025-04-25 01:29:01] (step=0084875) Train Loss: 6.3790, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38982/45917 [10:58:03<2:06:09, 1.09s/it][2025-04-25 01:29:28] (step=0084900) Train Loss: 6.3057, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39007/45917 [10:58:30<2:09:38, 1.13s/it][2025-04-25 01:29:56] (step=0084925) Train Loss: 6.2720, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39032/45917 [10:58:58<2:07:12, 1.11s/it][2025-04-25 01:30:24] (step=0084950) Train Loss: 6.2597, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39057/45917 [10:59:26<2:04:49, 1.09s/it][2025-04-25 01:30:51] (step=0084975) Train Loss: 6.2932, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39082/45917 [10:59:53<2:04:19, 1.09s/it][2025-04-25 01:31:19] (step=0085000) Train Loss: 6.3215, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39107/45917 [11:00:21<2:09:26, 1.14s/it][2025-04-25 01:31:47] (step=0085025) Train Loss: 6.2803, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39132/45917 [11:00:49<2:05:48, 1.11s/it][2025-04-25 01:32:15] (step=0085050) Train Loss: 6.3027, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39157/45917 [11:01:16<2:04:05, 1.10s/it][2025-04-25 01:32:42] (step=0085075) Train Loss: 6.2354, Train Steps/Sec: 0.91 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39182/45917 [11:01:44<2:03:09, 1.10s/it][2025-04-25 01:33:10] (step=0085100) Train Loss: 6.3276, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39207/45917 [11:02:12<2:05:37, 1.12s/it][2025-04-25 01:33:38] (step=0085125) Train Loss: 6.3194, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39232/45917 [11:02:40<2:04:16, 1.12s/it][2025-04-25 01:34:05] (step=0085150) Train Loss: 6.3059, Train Steps/Sec: 0.90 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39257/45917 [11:03:07<2:02:32, 1.10s/it][2025-04-25 01:34:33] (step=0085175) Train Loss: 6.3280, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39282/45917 [11:03:35<2:01:48, 1.10s/it][2025-04-25 01:35:01] (step=0085200) Train Loss: 6.3355, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39307/45917 [11:04:03<2:03:50, 1.12s/it][2025-04-25 01:35:29] (step=0085225) Train Loss: 6.3105, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39332/45917 [11:04:30<2:01:44, 1.11s/it][2025-04-25 01:35:56] (step=0085250) Train Loss: 6.2949, Train Steps/Sec: 0.91 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39357/45917 [11:04:58<1:59:09, 1.09s/it][2025-04-25 01:36:24] (step=0085275) Train Loss: 6.3008, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39382/45917 [11:05:26<1:58:19, 1.09s/it][2025-04-25 01:36:52] (step=0085300) Train Loss: 6.3230, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39407/45917 [11:05:54<2:01:31, 1.12s/it][2025-04-25 01:37:19] (step=0085325) Train Loss: 6.3097, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39432/45917 [11:06:21<2:00:21, 1.11s/it][2025-04-25 01:37:47] (step=0085350) Train Loss: 6.3685, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39457/45917 [11:06:49<1:59:20, 1.11s/it][2025-04-25 01:38:15] (step=0085375) Train Loss: 6.3726, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39482/45917 [11:07:17<1:57:39, 1.10s/it][2025-04-25 01:38:43] (step=0085400) Train Loss: 6.2958, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39507/45917 [11:07:45<2:00:31, 1.13s/it][2025-04-25 01:39:10] (step=0085425) Train Loss: 6.2952, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39532/45917 [11:08:12<1:57:33, 1.10s/it][2025-04-25 01:39:38] (step=0085450) Train Loss: 6.2994, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39557/45917 [11:08:40<1:57:10, 1.11s/it][2025-04-25 01:40:06] (step=0085475) Train Loss: 6.3279, Train Steps/Sec: 0.90 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39582/45917 [11:09:08<1:56:46, 1.11s/it][2025-04-25 01:40:34] (step=0085500) Train Loss: 6.2988, Train Steps/Sec: 0.89 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39607/45917 [11:09:36<1:58:08, 1.12s/it][2025-04-25 01:41:01] (step=0085525) Train Loss: 6.3568, Train Steps/Sec: 0.90 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39632/45917 [11:10:03<1:55:19, 1.10s/it][2025-04-25 01:41:29] (step=0085550) Train Loss: 6.2656, Train Steps/Sec: 0.90 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39657/45917 [11:10:31<1:53:56, 1.09s/it][2025-04-25 01:41:57] (step=0085575) Train Loss: 6.2918, Train Steps/Sec: 0.91 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39682/45917 [11:10:58<1:54:16, 1.10s/it][2025-04-25 01:42:24] (step=0085600) Train Loss: 6.3262, Train Steps/Sec: 0.90 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39707/45917 [11:11:26<1:56:26, 1.13s/it][2025-04-25 01:42:52] (step=0085625) Train Loss: 6.2828, Train Steps/Sec: 0.91 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39732/45917 [11:11:54<1:53:35, 1.10s/it][2025-04-25 01:43:20] (step=0085650) Train Loss: 6.2726, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39757/45917 [11:12:21<1:53:18, 1.10s/it][2025-04-25 01:43:47] (step=0085675) Train Loss: 6.2412, Train Steps/Sec: 0.91 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39782/45917 [11:12:49<1:52:14, 1.10s/it][2025-04-25 01:44:15] (step=0085700) Train Loss: 6.2661, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39807/45917 [11:13:17<1:54:10, 1.12s/it][2025-04-25 01:44:43] (step=0085725) Train Loss: 6.2798, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39832/45917 [11:13:45<1:52:36, 1.11s/it][2025-04-25 01:45:10] (step=0085750) Train Loss: 6.2791, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39857/45917 [11:14:12<1:50:34, 1.09s/it][2025-04-25 01:45:38] (step=0085775) Train Loss: 6.2548, Train Steps/Sec: 0.91 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39882/45917 [11:14:40<1:49:32, 1.09s/it][2025-04-25 01:46:06] (step=0085800) Train Loss: 6.3184, Train Steps/Sec: 0.90 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39907/45917 [11:15:08<1:52:58, 1.13s/it][2025-04-25 01:46:34] (step=0085825) Train Loss: 6.3161, Train Steps/Sec: 0.90 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39932/45917 [11:15:35<1:49:57, 1.10s/it][2025-04-25 01:47:01] (step=0085850) Train Loss: 6.3301, Train Steps/Sec: 0.91 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39957/45917 [11:16:03<1:49:55, 1.11s/it][2025-04-25 01:47:29] (step=0085875) Train Loss: 6.2779, Train Steps/Sec: 0.90 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39982/45917 [11:16:31<1:48:06, 1.09s/it][2025-04-25 01:47:57] (step=0085900) Train Loss: 6.3103, Train Steps/Sec: 0.90 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 40007/45917 [11:16:59<1:50:59, 1.13s/it][2025-04-25 01:48:24] (step=0085925) Train Loss: 6.3249, Train Steps/Sec: 0.90 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 40032/45917 [11:17:26<1:49:14, 1.11s/it][2025-04-25 01:48:52] (step=0085950) Train Loss: 6.3075, Train Steps/Sec: 0.90 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 40057/45917 [11:17:54<1:47:32, 1.10s/it][2025-04-25 01:49:20] (step=0085975) Train Loss: 6.3152, Train Steps/Sec: 0.90 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 40082/45917 [11:18:22<1:47:20, 1.10s/it][2025-04-25 01:49:48] (step=0086000) Train Loss: 6.2855, Train Steps/Sec: 0.90 +[2025-04-25 01:49:48] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-25 01:49:48] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-25 01:49:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.44s/it] +[2025-04-25 01:54:33] Finish Eval in 86000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 58.97s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 01:54:55] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0086000.pt +[2025-04-25 01:54:57] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0084000.pt + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 40107/45917 [11:23:59<1:51:41, 1.15s/it][2025-04-25 01:55:25] (step=0086025) Train Loss: 6.2775, Train Steps/Sec: 0.07 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 40132/45917 [11:24:27<1:46:51, 1.11s/it][2025-04-25 01:55:53] (step=0086050) Train Loss: 6.2732, Train Steps/Sec: 0.90 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40157/45917 [11:24:54<1:45:46, 1.10s/it][2025-04-25 01:56:20] (step=0086075) Train Loss: 6.3008, Train Steps/Sec: 0.91 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40182/45917 [11:25:22<1:46:36, 1.12s/it][2025-04-25 01:56:48] (step=0086100) Train Loss: 6.3039, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40207/45917 [11:25:50<1:46:30, 1.12s/it][2025-04-25 01:57:16] (step=0086125) Train Loss: 6.3156, Train Steps/Sec: 0.91 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40232/45917 [11:26:17<1:45:02, 1.11s/it][2025-04-25 01:57:43] (step=0086150) Train Loss: 6.3149, Train Steps/Sec: 0.90 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40257/45917 [11:26:45<1:43:17, 1.10s/it][2025-04-25 01:58:11] (step=0086175) Train Loss: 6.3013, Train Steps/Sec: 0.90 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40282/45917 [11:27:13<1:42:53, 1.10s/it][2025-04-25 01:58:39] (step=0086200) Train Loss: 6.2565, Train Steps/Sec: 0.90 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 40307/45917 [11:27:41<1:44:47, 1.12s/it][2025-04-25 01:59:06] (step=0086225) Train Loss: 6.2862, Train Steps/Sec: 0.91 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 40332/45917 [11:28:08<1:43:01, 1.11s/it][2025-04-25 01:59:34] (step=0086250) Train Loss: 6.2950, Train Steps/Sec: 0.91 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 40357/45917 [11:28:36<1:42:17, 1.10s/it][2025-04-25 02:00:02] (step=0086275) Train Loss: 6.3030, Train Steps/Sec: 0.90 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 40382/45917 [11:29:04<1:41:20, 1.10s/it][2025-04-25 02:00:29] (step=0086300) Train Loss: 6.2941, Train Steps/Sec: 0.90 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 40407/45917 [11:29:32<1:44:35, 1.14s/it][2025-04-25 02:00:57] (step=0086325) Train Loss: 6.3046, Train Steps/Sec: 0.90 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 40432/45917 [11:29:59<1:41:40, 1.11s/it][2025-04-25 02:01:25] (step=0086350) Train Loss: 6.2654, Train Steps/Sec: 0.90 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 40457/45917 [11:30:27<1:41:17, 1.11s/it][2025-04-25 02:01:53] (step=0086375) Train Loss: 6.3108, Train Steps/Sec: 0.90 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40482/45917 [11:30:55<1:39:02, 1.09s/it][2025-04-25 02:02:21] (step=0086400) Train Loss: 6.3832, Train Steps/Sec: 0.90 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40507/45917 [11:31:23<1:42:14, 1.13s/it][2025-04-25 02:02:48] (step=0086425) Train Loss: 6.3336, Train Steps/Sec: 0.90 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40532/45917 [11:31:50<1:38:50, 1.10s/it][2025-04-25 02:03:16] (step=0086450) Train Loss: 6.2664, Train Steps/Sec: 0.91 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40557/45917 [11:32:18<1:38:26, 1.10s/it][2025-04-25 02:03:44] (step=0086475) Train Loss: 6.3266, Train Steps/Sec: 0.90 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40582/45917 [11:32:46<1:38:14, 1.10s/it][2025-04-25 02:04:11] (step=0086500) Train Loss: 6.2752, Train Steps/Sec: 0.90 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40607/45917 [11:33:13<1:39:03, 1.12s/it][2025-04-25 02:04:39] (step=0086525) Train Loss: 6.3390, Train Steps/Sec: 0.91 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40632/45917 [11:33:41<1:36:42, 1.10s/it][2025-04-25 02:05:07] (step=0086550) Train Loss: 6.3030, Train Steps/Sec: 0.90 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 40657/45917 [11:34:09<1:36:42, 1.10s/it][2025-04-25 02:05:34] (step=0086575) Train Loss: 6.2414, Train Steps/Sec: 0.90 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 40682/45917 [11:34:36<1:35:12, 1.09s/it][2025-04-25 02:06:02] (step=0086600) Train Loss: 6.3346, Train Steps/Sec: 0.90 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 40707/45917 [11:35:04<1:37:01, 1.12s/it][2025-04-25 02:06:30] (step=0086625) Train Loss: 6.3373, Train Steps/Sec: 0.91 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 40732/45917 [11:35:32<1:35:54, 1.11s/it][2025-04-25 02:06:57] (step=0086650) Train Loss: 6.3279, Train Steps/Sec: 0.90 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 40757/45917 [11:36:00<1:34:51, 1.10s/it][2025-04-25 02:07:25] (step=0086675) Train Loss: 6.3092, Train Steps/Sec: 0.90 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 40782/45917 [11:36:27<1:34:03, 1.10s/it][2025-04-25 02:07:53] (step=0086700) Train Loss: 6.2794, Train Steps/Sec: 0.90 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40807/45917 [11:36:55<1:36:21, 1.13s/it][2025-04-25 02:08:21] (step=0086725) Train Loss: 6.2821, Train Steps/Sec: 0.90 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40832/45917 [11:37:23<1:34:12, 1.11s/it][2025-04-25 02:08:49] (step=0086750) Train Loss: 6.2718, Train Steps/Sec: 0.90 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40857/45917 [11:37:56<1:33:11, 1.10s/it][2025-04-25 02:09:22] (step=0086775) Train Loss: 6.3062, Train Steps/Sec: 0.75 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40882/45917 [11:38:24<1:31:45, 1.09s/it][2025-04-25 02:09:50] (step=0086800) Train Loss: 6.3488, Train Steps/Sec: 0.90 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40907/45917 [11:38:52<1:34:40, 1.13s/it][2025-04-25 02:10:17] (step=0086825) Train Loss: 6.3040, Train Steps/Sec: 0.91 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40932/45917 [11:39:19<1:32:04, 1.11s/it][2025-04-25 02:10:45] (step=0086850) Train Loss: 6.2927, Train Steps/Sec: 0.91 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40957/45917 [11:39:47<1:30:45, 1.10s/it][2025-04-25 02:11:13] (step=0086875) Train Loss: 6.3427, Train Steps/Sec: 0.90 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 40982/45917 [11:40:15<1:30:00, 1.09s/it][2025-04-25 02:11:40] (step=0086900) Train Loss: 6.3092, Train Steps/Sec: 0.90 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 41007/45917 [11:40:42<1:31:09, 1.11s/it][2025-04-25 02:12:08] (step=0086925) Train Loss: 6.2936, Train Steps/Sec: 0.90 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 41032/45917 [11:41:10<1:30:06, 1.11s/it][2025-04-25 02:12:36] (step=0086950) Train Loss: 6.3117, Train Steps/Sec: 0.91 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 41057/45917 [11:41:38<1:29:29, 1.10s/it][2025-04-25 02:13:04] (step=0086975) Train Loss: 6.2978, Train Steps/Sec: 0.90 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 41082/45917 [11:42:05<1:28:14, 1.10s/it][2025-04-25 02:13:31] (step=0087000) Train Loss: 6.3165, Train Steps/Sec: 0.90 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 41107/45917 [11:42:33<1:30:57, 1.13s/it][2025-04-25 02:13:59] (step=0087025) Train Loss: 6.2955, Train Steps/Sec: 0.90 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 41132/45917 [11:43:01<1:29:02, 1.12s/it][2025-04-25 02:14:27] (step=0087050) Train Loss: 6.2856, Train Steps/Sec: 0.90 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 41157/45917 [11:43:29<1:27:21, 1.10s/it][2025-04-25 02:14:55] (step=0087075) Train Loss: 6.2954, Train Steps/Sec: 0.90 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 41182/45917 [11:43:57<1:26:27, 1.10s/it][2025-04-25 02:15:22] (step=0087100) Train Loss: 6.2920, Train Steps/Sec: 0.90 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 41207/45917 [11:44:24<1:27:44, 1.12s/it][2025-04-25 02:15:50] (step=0087125) Train Loss: 6.2945, Train Steps/Sec: 0.91 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 41232/45917 [11:44:52<1:26:51, 1.11s/it][2025-04-25 02:16:18] (step=0087150) Train Loss: 6.3409, Train Steps/Sec: 0.90 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 41257/45917 [11:45:20<1:25:31, 1.10s/it][2025-04-25 02:16:45] (step=0087175) Train Loss: 6.2625, Train Steps/Sec: 0.90 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 41282/45917 [11:45:47<1:24:04, 1.09s/it][2025-04-25 02:17:13] (step=0087200) Train Loss: 6.2747, Train Steps/Sec: 0.90 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 41307/45917 [11:46:15<1:26:18, 1.12s/it][2025-04-25 02:17:41] (step=0087225) Train Loss: 6.2830, Train Steps/Sec: 0.90 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 41332/45917 [11:46:43<1:24:48, 1.11s/it][2025-04-25 02:18:09] (step=0087250) Train Loss: 6.2774, Train Steps/Sec: 0.91 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 41357/45917 [11:47:11<1:24:23, 1.11s/it][2025-04-25 02:18:36] (step=0087275) Train Loss: 6.3041, Train Steps/Sec: 0.90 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 41382/45917 [11:47:38<1:22:34, 1.09s/it][2025-04-25 02:19:04] (step=0087300) Train Loss: 6.3082, Train Steps/Sec: 0.90 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 41407/45917 [11:48:06<1:24:16, 1.12s/it][2025-04-25 02:19:32] (step=0087325) Train Loss: 6.3364, Train Steps/Sec: 0.90 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 41432/45917 [11:48:34<1:23:10, 1.11s/it][2025-04-25 02:19:59] (step=0087350) Train Loss: 6.2994, Train Steps/Sec: 0.90 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 41457/45917 [11:49:01<1:21:49, 1.10s/it][2025-04-25 02:20:27] (step=0087375) Train Loss: 6.2704, Train Steps/Sec: 0.90 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 41482/45917 [11:49:29<1:20:49, 1.09s/it][2025-04-25 02:20:55] (step=0087400) Train Loss: 6.2622, Train Steps/Sec: 0.90 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 41507/45917 [11:49:57<1:23:03, 1.13s/it][2025-04-25 02:21:23] (step=0087425) Train Loss: 6.3095, Train Steps/Sec: 0.90 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 41532/45917 [11:50:24<1:21:15, 1.11s/it][2025-04-25 02:21:50] (step=0087450) Train Loss: 6.2854, Train Steps/Sec: 0.91 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 41557/45917 [11:50:52<1:18:56, 1.09s/it][2025-04-25 02:22:18] (step=0087475) Train Loss: 6.2772, Train Steps/Sec: 0.90 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 41582/45917 [11:51:20<1:18:24, 1.09s/it][2025-04-25 02:22:46] (step=0087500) Train Loss: 6.3088, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 41607/45917 [11:51:48<1:20:50, 1.13s/it][2025-04-25 02:23:13] (step=0087525) Train Loss: 6.3019, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 41632/45917 [11:52:15<1:19:08, 1.11s/it][2025-04-25 02:23:41] (step=0087550) Train Loss: 6.3003, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 41657/45917 [11:52:43<1:17:14, 1.09s/it][2025-04-25 02:24:09] (step=0087575) Train Loss: 6.2610, Train Steps/Sec: 0.91 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 41682/45917 [11:53:11<1:17:25, 1.10s/it][2025-04-25 02:24:37] (step=0087600) Train Loss: 6.2639, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 41707/45917 [11:53:39<1:19:02, 1.13s/it][2025-04-25 02:25:04] (step=0087625) Train Loss: 6.3034, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 41732/45917 [11:54:06<1:17:53, 1.12s/it][2025-04-25 02:25:32] (step=0087650) Train Loss: 6.2550, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 41757/45917 [11:54:34<1:17:32, 1.12s/it][2025-04-25 02:26:00] (step=0087675) Train Loss: 6.2987, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 41782/45917 [11:55:02<1:15:17, 1.09s/it][2025-04-25 02:26:28] (step=0087700) Train Loss: 6.2837, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 41807/45917 [11:55:30<1:17:48, 1.14s/it][2025-04-25 02:26:56] (step=0087725) Train Loss: 6.3282, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 41832/45917 [11:55:57<1:15:59, 1.12s/it][2025-04-25 02:27:23] (step=0087750) Train Loss: 6.2780, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 41857/45917 [11:56:25<1:14:51, 1.11s/it][2025-04-25 02:27:51] (step=0087775) Train Loss: 6.3237, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 41882/45917 [11:56:53<1:13:46, 1.10s/it][2025-04-25 02:28:19] (step=0087800) Train Loss: 6.3002, Train Steps/Sec: 0.90 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 41907/45917 [11:57:21<1:15:07, 1.12s/it][2025-04-25 02:28:47] (step=0087825) Train Loss: 6.2739, Train Steps/Sec: 0.90 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 41932/45917 [11:57:48<1:13:39, 1.11s/it][2025-04-25 02:29:14] (step=0087850) Train Loss: 6.3164, Train Steps/Sec: 0.90 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 41957/45917 [11:58:16<1:12:14, 1.09s/it][2025-04-25 02:29:42] (step=0087875) Train Loss: 6.2999, Train Steps/Sec: 0.91 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 41982/45917 [11:58:44<1:12:09, 1.10s/it][2025-04-25 02:30:10] (step=0087900) Train Loss: 6.3084, Train Steps/Sec: 0.90 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42007/45917 [11:59:11<1:13:33, 1.13s/it][2025-04-25 02:30:37] (step=0087925) Train Loss: 6.3223, Train Steps/Sec: 0.90 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 42032/45917 [11:59:39<1:11:06, 1.10s/it][2025-04-25 02:31:05] (step=0087950) Train Loss: 6.2876, Train Steps/Sec: 0.91 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 42057/45917 [12:00:07<1:10:56, 1.10s/it][2025-04-25 02:31:32] (step=0087975) Train Loss: 6.3445, Train Steps/Sec: 0.90 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 42082/45917 [12:00:34<1:10:17, 1.10s/it][2025-04-25 02:32:00] (step=0088000) Train Loss: 6.3103, Train Steps/Sec: 0.90 +[2025-04-25 02:32:00] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-25 02:32:00] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-25 02:32:00] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:57<00:00, 59.10s/it] +[2025-04-25 02:36:45] Finish Eval in 88000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:56<00:00, 58.64s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 02:37:07] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0088000.pt +[2025-04-25 02:37:09] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0086000.pt + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42107/45917 [12:06:11<1:12:25, 1.14s/it][2025-04-25 02:37:37] (step=0088025) Train Loss: 6.3040, Train Steps/Sec: 0.07 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42132/45917 [12:06:39<1:09:57, 1.11s/it][2025-04-25 02:38:05] (step=0088050) Train Loss: 6.3146, Train Steps/Sec: 0.90 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 42157/45917 [12:07:06<1:09:30, 1.11s/it][2025-04-25 02:38:32] (step=0088075) Train Loss: 6.2700, Train Steps/Sec: 0.91 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 42182/45917 [12:07:34<1:08:27, 1.10s/it][2025-04-25 02:39:00] (step=0088100) Train Loss: 6.2805, Train Steps/Sec: 0.89 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 42207/45917 [12:08:02<1:09:06, 1.12s/it][2025-04-25 02:39:28] (step=0088125) Train Loss: 6.3108, Train Steps/Sec: 0.91 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 42232/45917 [12:08:30<1:07:21, 1.10s/it][2025-04-25 02:39:55] (step=0088150) Train Loss: 6.3120, Train Steps/Sec: 0.91 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 42257/45917 [12:08:57<1:07:00, 1.10s/it][2025-04-25 02:40:23] (step=0088175) Train Loss: 6.3094, Train Steps/Sec: 0.90 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 42282/45917 [12:09:25<1:06:02, 1.09s/it][2025-04-25 02:40:51] (step=0088200) Train Loss: 6.2554, Train Steps/Sec: 0.89 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42307/45917 [12:09:53<1:07:49, 1.13s/it][2025-04-25 02:41:19] (step=0088225) Train Loss: 6.2714, Train Steps/Sec: 0.90 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42332/45917 [12:10:21<1:06:11, 1.11s/it][2025-04-25 02:41:47] (step=0088250) Train Loss: 6.3072, Train Steps/Sec: 0.90 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 42357/45917 [12:10:48<1:05:29, 1.10s/it][2025-04-25 02:42:14] (step=0088275) Train Loss: 6.3092, Train Steps/Sec: 0.91 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 42382/45917 [12:11:16<1:04:38, 1.10s/it][2025-04-25 02:42:42] (step=0088300) Train Loss: 6.2877, Train Steps/Sec: 0.90 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 42407/45917 [12:11:44<1:06:00, 1.13s/it][2025-04-25 02:43:10] (step=0088325) Train Loss: 6.2977, Train Steps/Sec: 0.91 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42432/45917 [12:12:11<1:04:18, 1.11s/it][2025-04-25 02:43:37] (step=0088350) Train Loss: 6.2850, Train Steps/Sec: 0.90 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42457/45917 [12:12:39<1:03:17, 1.10s/it][2025-04-25 02:44:05] (step=0088375) Train Loss: 6.3244, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 42482/45917 [12:13:07<1:02:56, 1.10s/it][2025-04-25 02:44:33] (step=0088400) Train Loss: 6.2553, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 42507/45917 [12:13:35<1:04:02, 1.13s/it][2025-04-25 02:45:01] (step=0088425) Train Loss: 6.3050, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 42532/45917 [12:14:02<1:02:04, 1.10s/it][2025-04-25 02:45:28] (step=0088450) Train Loss: 6.3403, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 42557/45917 [12:14:30<1:00:57, 1.09s/it][2025-04-25 02:45:56] (step=0088475) Train Loss: 6.2941, Train Steps/Sec: 0.90 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 42582/45917 [12:14:58<1:01:17, 1.10s/it][2025-04-25 02:46:24] (step=0088500) Train Loss: 6.3056, Train Steps/Sec: 0.90 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 42607/45917 [12:15:26<1:01:39, 1.12s/it][2025-04-25 02:46:51] (step=0088525) Train Loss: 6.3169, Train Steps/Sec: 0.91 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42632/45917 [12:15:53<1:01:01, 1.11s/it][2025-04-25 02:47:19] (step=0088550) Train Loss: 6.2624, Train Steps/Sec: 0.90 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 42657/45917 [12:16:21<59:39, 1.10s/it][2025-04-25 02:47:47] (step=0088575) Train Loss: 6.3011, Train Steps/Sec: 0.90 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 42682/45917 [12:16:49<59:19, 1.10s/it][2025-04-25 02:48:14] (step=0088600) Train Loss: 6.3055, Train Steps/Sec: 0.90 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 42707/45917 [12:17:16<1:00:10, 1.12s/it][2025-04-25 02:48:42] (step=0088625) Train Loss: 6.3058, Train Steps/Sec: 0.90 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 42732/45917 [12:17:44<58:50, 1.11s/it][2025-04-25 02:49:10] (step=0088650) Train Loss: 6.2882, Train Steps/Sec: 0.90 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 42757/45917 [12:18:12<58:06, 1.10s/it][2025-04-25 02:49:38] (step=0088675) Train Loss: 6.2599, Train Steps/Sec: 0.90 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 42782/45917 [12:18:40<56:53, 1.09s/it][2025-04-25 02:50:05] (step=0088700) Train Loss: 6.3065, Train Steps/Sec: 0.90 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42807/45917 [12:19:07<58:30, 1.13s/it][2025-04-25 02:50:33] (step=0088725) Train Loss: 6.3309, Train Steps/Sec: 0.90 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 42832/45917 [12:19:35<56:59, 1.11s/it][2025-04-25 02:51:01] (step=0088750) Train Loss: 6.3137, Train Steps/Sec: 0.90 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 42857/45917 [12:20:03<56:17, 1.10s/it][2025-04-25 02:51:29] (step=0088775) Train Loss: 6.2965, Train Steps/Sec: 0.90 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 42882/45917 [12:20:31<55:17, 1.09s/it][2025-04-25 02:51:56] (step=0088800) Train Loss: 6.2905, Train Steps/Sec: 0.90 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 42907/45917 [12:20:58<56:22, 1.12s/it][2025-04-25 02:52:24] (step=0088825) Train Loss: 6.2581, Train Steps/Sec: 0.90 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 42932/45917 [12:21:26<55:32, 1.12s/it][2025-04-25 02:52:52] (step=0088850) Train Loss: 6.3276, Train Steps/Sec: 0.90 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 42957/45917 [12:21:54<54:06, 1.10s/it][2025-04-25 02:53:20] (step=0088875) Train Loss: 6.3172, Train Steps/Sec: 0.90 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 42982/45917 [12:22:22<54:23, 1.11s/it][2025-04-25 02:53:47] (step=0088900) Train Loss: 6.2845, Train Steps/Sec: 0.90 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 43007/45917 [12:22:49<54:16, 1.12s/it][2025-04-25 02:54:15] (step=0088925) Train Loss: 6.2641, Train Steps/Sec: 0.90 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 43032/45917 [12:23:22<59:01, 1.23s/it][2025-04-25 02:54:48] (step=0088950) Train Loss: 6.3168, Train Steps/Sec: 0.77 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43057/45917 [12:23:50<52:15, 1.10s/it][2025-04-25 02:55:15] (step=0088975) Train Loss: 6.2634, Train Steps/Sec: 0.91 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43082/45917 [12:24:17<51:50, 1.10s/it][2025-04-25 02:55:43] (step=0089000) Train Loss: 6.3062, Train Steps/Sec: 0.90 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 43107/45917 [12:24:50<1:04:22, 1.37s/it][2025-04-25 02:56:16] (step=0089025) Train Loss: 6.3064, Train Steps/Sec: 0.77 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 43132/45917 [12:25:18<51:42, 1.11s/it][2025-04-25 02:56:44] (step=0089050) Train Loss: 6.2868, Train Steps/Sec: 0.90 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 43157/45917 [12:25:51<51:20, 1.12s/it][2025-04-25 02:57:16] (step=0089075) Train Loss: 6.2907, Train Steps/Sec: 0.76 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 43182/45917 [12:26:19<49:59, 1.10s/it][2025-04-25 02:57:44] (step=0089100) Train Loss: 6.3028, Train Steps/Sec: 0.90 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 43207/45917 [12:26:46<50:55, 1.13s/it][2025-04-25 02:58:12] (step=0089125) Train Loss: 6.3534, Train Steps/Sec: 0.90 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 43232/45917 [12:27:14<49:12, 1.10s/it][2025-04-25 02:58:40] (step=0089150) Train Loss: 6.2347, Train Steps/Sec: 0.91 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43257/45917 [12:27:42<48:22, 1.09s/it][2025-04-25 02:59:07] (step=0089175) Train Loss: 6.2856, Train Steps/Sec: 0.90 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43282/45917 [12:28:09<48:25, 1.10s/it][2025-04-25 02:59:35] (step=0089200) Train Loss: 6.3176, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 43307/45917 [12:28:37<48:48, 1.12s/it][2025-04-25 03:00:03] (step=0089225) Train Loss: 6.2587, Train Steps/Sec: 0.91 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 43332/45917 [12:29:05<48:21, 1.12s/it][2025-04-25 03:00:31] (step=0089250) Train Loss: 6.2923, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 43357/45917 [12:29:32<47:09, 1.11s/it][2025-04-25 03:00:58] (step=0089275) Train Loss: 6.2808, Train Steps/Sec: 0.90 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43382/45917 [12:30:00<46:45, 1.11s/it][2025-04-25 03:01:26] (step=0089300) Train Loss: 6.3330, Train Steps/Sec: 0.90 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43407/45917 [12:30:33<47:09, 1.13s/it][2025-04-25 03:01:59] (step=0089325) Train Loss: 6.3319, Train Steps/Sec: 0.77 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 43432/45917 [12:31:01<45:57, 1.11s/it][2025-04-25 03:02:26] (step=0089350) Train Loss: 6.3387, Train Steps/Sec: 0.90 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 43457/45917 [12:31:28<45:05, 1.10s/it][2025-04-25 03:02:54] (step=0089375) Train Loss: 6.2629, Train Steps/Sec: 0.91 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 43482/45917 [12:32:01<44:36, 1.10s/it][2025-04-25 03:03:27] (step=0089400) Train Loss: 6.3081, Train Steps/Sec: 0.76 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 43507/45917 [12:32:29<45:19, 1.13s/it][2025-04-25 03:03:55] (step=0089425) Train Loss: 6.3291, Train Steps/Sec: 0.90 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 43532/45917 [12:32:57<43:54, 1.10s/it][2025-04-25 03:04:22] (step=0089450) Train Loss: 6.3304, Train Steps/Sec: 0.91 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43557/45917 [12:33:30<1:27:15, 2.22s/it][2025-04-25 03:04:55] (step=0089475) Train Loss: 6.3759, Train Steps/Sec: 0.76 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43582/45917 [12:33:57<42:34, 1.09s/it][2025-04-25 03:05:23] (step=0089500) Train Loss: 6.2596, Train Steps/Sec: 0.90 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43607/45917 [12:34:25<42:59, 1.12s/it][2025-04-25 03:05:51] (step=0089525) Train Loss: 6.2852, Train Steps/Sec: 0.91 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 43632/45917 [12:34:53<42:12, 1.11s/it][2025-04-25 03:06:18] (step=0089550) Train Loss: 6.3186, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 43657/45917 [12:35:20<41:12, 1.09s/it][2025-04-25 03:06:46] (step=0089575) Train Loss: 6.2623, Train Steps/Sec: 0.91 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 43682/45917 [12:35:48<40:51, 1.10s/it][2025-04-25 03:07:14] (step=0089600) Train Loss: 6.3035, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43707/45917 [12:36:16<41:28, 1.13s/it][2025-04-25 03:07:41] (step=0089625) Train Loss: 6.3085, Train Steps/Sec: 0.91 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 43732/45917 [12:36:43<40:39, 1.12s/it][2025-04-25 03:08:09] (step=0089650) Train Loss: 6.3141, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 43757/45917 [12:37:11<39:37, 1.10s/it][2025-04-25 03:08:37] (step=0089675) Train Loss: 6.2930, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 43782/45917 [12:37:39<39:04, 1.10s/it][2025-04-25 03:09:05] (step=0089700) Train Loss: 6.3078, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 43807/45917 [12:38:06<39:50, 1.13s/it][2025-04-25 03:09:32] (step=0089725) Train Loss: 6.3394, Train Steps/Sec: 0.90 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 43832/45917 [12:38:34<38:22, 1.10s/it][2025-04-25 03:10:00] (step=0089750) Train Loss: 6.3161, Train Steps/Sec: 0.90 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 43857/45917 [12:39:02<37:10, 1.08s/it][2025-04-25 03:10:28] (step=0089775) Train Loss: 6.3034, Train Steps/Sec: 0.91 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 43882/45917 [12:39:29<37:15, 1.10s/it][2025-04-25 03:10:55] (step=0089800) Train Loss: 6.3170, Train Steps/Sec: 0.90 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 43907/45917 [12:39:57<37:48, 1.13s/it][2025-04-25 03:11:23] (step=0089825) Train Loss: 6.3449, Train Steps/Sec: 0.91 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43932/45917 [12:40:25<36:36, 1.11s/it][2025-04-25 03:11:51] (step=0089850) Train Loss: 6.2610, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 43957/45917 [12:40:53<35:57, 1.10s/it][2025-04-25 03:12:18] (step=0089875) Train Loss: 6.3280, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 43982/45917 [12:41:20<35:32, 1.10s/it][2025-04-25 03:12:46] (step=0089900) Train Loss: 6.2787, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 44007/45917 [12:41:48<35:52, 1.13s/it][2025-04-25 03:13:14] (step=0089925) Train Loss: 6.3293, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 44032/45917 [12:42:16<34:48, 1.11s/it][2025-04-25 03:13:41] (step=0089950) Train Loss: 6.3107, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 44057/45917 [12:42:43<34:13, 1.10s/it][2025-04-25 03:14:09] (step=0089975) Train Loss: 6.3437, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 44082/45917 [12:43:11<33:23, 1.09s/it][2025-04-25 03:14:37] (step=0090000) Train Loss: 6.2817, Train Steps/Sec: 0.90 +[2025-04-25 03:14:37] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-25 03:14:37] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-25 03:14:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:59<00:00, 59.70s/it] +[2025-04-25 03:19:24] Finish Eval in 90000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:58<00:00, 59.18s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 03:19:45] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0090000.pt +[2025-04-25 03:19:47] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/0088000.pt + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 44107/45917 [12:48:50<34:35, 1.15s/it][2025-04-25 03:20:15] (step=0090025) Train Loss: 6.2824, Train Steps/Sec: 0.07 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 44132/45917 [12:49:18<33:51, 1.14s/it][2025-04-25 03:20:43] (step=0090050) Train Loss: 6.2990, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 44157/45917 [12:49:45<32:19, 1.10s/it][2025-04-25 03:21:11] (step=0090075) Train Loss: 6.2982, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 44182/45917 [12:50:13<31:40, 1.10s/it][2025-04-25 03:21:39] (step=0090100) Train Loss: 6.2876, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44207/45917 [12:50:41<32:24, 1.14s/it][2025-04-25 03:22:07] (step=0090125) Train Loss: 6.2717, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44232/45917 [12:51:09<31:08, 1.11s/it][2025-04-25 03:22:34] (step=0090150) Train Loss: 6.3336, Train Steps/Sec: 0.90 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 44257/45917 [12:51:36<30:24, 1.10s/it][2025-04-25 03:23:02] (step=0090175) Train Loss: 6.2825, Train Steps/Sec: 0.91 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 44282/45917 [12:52:04<30:00, 1.10s/it][2025-04-25 03:23:30] (step=0090200) Train Loss: 6.2849, Train Steps/Sec: 0.90 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 44307/45917 [12:52:32<30:22, 1.13s/it][2025-04-25 03:23:57] (step=0090225) Train Loss: 6.2645, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 44332/45917 [12:52:59<29:23, 1.11s/it][2025-04-25 03:24:25] (step=0090250) Train Loss: 6.2930, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 44357/45917 [12:53:27<28:36, 1.10s/it][2025-04-25 03:24:53] (step=0090275) Train Loss: 6.2673, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 44382/45917 [12:53:55<28:20, 1.11s/it][2025-04-25 03:25:21] (step=0090300) Train Loss: 6.2552, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 44407/45917 [12:54:23<28:28, 1.13s/it][2025-04-25 03:25:48] (step=0090325) Train Loss: 6.3100, Train Steps/Sec: 0.91 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 44432/45917 [12:54:50<27:21, 1.11s/it][2025-04-25 03:26:16] (step=0090350) Train Loss: 6.3062, Train Steps/Sec: 0.91 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 44457/45917 [12:55:18<26:48, 1.10s/it][2025-04-25 03:26:44] (step=0090375) Train Loss: 6.2920, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 44482/45917 [12:55:45<25:58, 1.09s/it][2025-04-25 03:27:11] (step=0090400) Train Loss: 6.3121, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 44507/45917 [12:56:13<26:25, 1.12s/it][2025-04-25 03:27:39] (step=0090425) Train Loss: 6.2936, Train Steps/Sec: 0.91 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44532/45917 [12:56:41<25:29, 1.10s/it][2025-04-25 03:28:06] (step=0090450) Train Loss: 6.3431, Train Steps/Sec: 0.91 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44557/45917 [12:57:08<24:48, 1.09s/it][2025-04-25 03:28:34] (step=0090475) Train Loss: 6.3088, Train Steps/Sec: 0.90 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 44582/45917 [12:57:36<24:19, 1.09s/it][2025-04-25 03:29:02] (step=0090500) Train Loss: 6.2637, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 44607/45917 [12:58:04<24:43, 1.13s/it][2025-04-25 03:29:30] (step=0090525) Train Loss: 6.3165, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 44632/45917 [12:58:32<23:48, 1.11s/it][2025-04-25 03:29:57] (step=0090550) Train Loss: 6.3132, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 44657/45917 [12:58:59<23:20, 1.11s/it][2025-04-25 03:30:25] (step=0090575) Train Loss: 6.2973, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 44682/45917 [12:59:27<22:28, 1.09s/it][2025-04-25 03:30:53] (step=0090600) Train Loss: 6.3208, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 44707/45917 [12:59:55<22:44, 1.13s/it][2025-04-25 03:31:21] (step=0090625) Train Loss: 6.2988, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 44732/45917 [13:00:23<21:59, 1.11s/it][2025-04-25 03:31:48] (step=0090650) Train Loss: 6.3348, Train Steps/Sec: 0.90 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 44757/45917 [13:00:50<21:12, 1.10s/it][2025-04-25 03:32:16] (step=0090675) Train Loss: 6.2510, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 44782/45917 [13:01:18<20:43, 1.10s/it][2025-04-25 03:32:44] (step=0090700) Train Loss: 6.3162, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 44807/45917 [13:01:46<20:49, 1.13s/it][2025-04-25 03:33:12] (step=0090725) Train Loss: 6.3292, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 44832/45917 [13:02:14<20:05, 1.11s/it][2025-04-25 03:33:39] (step=0090750) Train Loss: 6.2796, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44857/45917 [13:02:41<19:26, 1.10s/it][2025-04-25 03:34:07] (step=0090775) Train Loss: 6.3029, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 44882/45917 [13:03:09<18:46, 1.09s/it][2025-04-25 03:34:35] (step=0090800) Train Loss: 6.2559, Train Steps/Sec: 0.90 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 44907/45917 [13:03:37<18:58, 1.13s/it][2025-04-25 03:35:02] (step=0090825) Train Loss: 6.2947, Train Steps/Sec: 0.91 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 44932/45917 [13:04:04<18:13, 1.11s/it][2025-04-25 03:35:30] (step=0090850) Train Loss: 6.2756, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 44957/45917 [13:04:32<17:35, 1.10s/it][2025-04-25 03:35:58] (step=0090875) Train Loss: 6.3191, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 44982/45917 [13:05:00<17:06, 1.10s/it][2025-04-25 03:36:26] (step=0090900) Train Loss: 6.2892, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 45007/45917 [13:05:28<17:04, 1.13s/it][2025-04-25 03:36:53] (step=0090925) Train Loss: 6.3120, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 45032/45917 [13:05:55<16:18, 1.11s/it][2025-04-25 03:37:21] (step=0090950) Train Loss: 6.3054, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 45057/45917 [13:06:23<15:48, 1.10s/it][2025-04-25 03:37:49] (step=0090975) Train Loss: 6.3085, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 45082/45917 [13:06:51<15:12, 1.09s/it][2025-04-25 03:38:17] (step=0091000) Train Loss: 6.2819, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 45107/45917 [13:07:18<15:12, 1.13s/it][2025-04-25 03:38:44] (step=0091025) Train Loss: 6.3684, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 45132/45917 [13:07:46<14:29, 1.11s/it][2025-04-25 03:39:12] (step=0091050) Train Loss: 6.3019, Train Steps/Sec: 0.91 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 45157/45917 [13:08:14<14:01, 1.11s/it][2025-04-25 03:39:40] (step=0091075) Train Loss: 6.3376, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 45182/45917 [13:08:41<13:21, 1.09s/it][2025-04-25 03:40:07] (step=0091100) Train Loss: 6.2707, Train Steps/Sec: 0.90 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 45207/45917 [13:09:09<13:13, 1.12s/it][2025-04-25 03:40:35] (step=0091125) Train Loss: 6.2692, Train Steps/Sec: 0.90 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 45232/45917 [13:09:37<12:39, 1.11s/it][2025-04-25 03:41:03] (step=0091150) Train Loss: 6.3318, Train Steps/Sec: 0.90 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 45257/45917 [13:10:05<12:17, 1.12s/it][2025-04-25 03:41:31] (step=0091175) Train Loss: 6.3087, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 45282/45917 [13:10:33<11:40, 1.10s/it][2025-04-25 03:41:58] (step=0091200) Train Loss: 6.3040, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 45307/45917 [13:11:00<11:28, 1.13s/it][2025-04-25 03:42:26] (step=0091225) Train Loss: 6.2815, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 45332/45917 [13:11:28<10:49, 1.11s/it][2025-04-25 03:42:54] (step=0091250) Train Loss: 6.2902, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 45357/45917 [13:11:56<10:17, 1.10s/it][2025-04-25 03:43:22] (step=0091275) Train Loss: 6.3302, Train Steps/Sec: 0.91 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 45382/45917 [13:12:24<09:46, 1.10s/it][2025-04-25 03:43:49] (step=0091300) Train Loss: 6.3278, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 45407/45917 [13:12:51<09:36, 1.13s/it][2025-04-25 03:44:17] (step=0091325) Train Loss: 6.3469, Train Steps/Sec: 0.91 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 45432/45917 [13:13:19<08:56, 1.11s/it][2025-04-25 03:44:45] (step=0091350) Train Loss: 6.2995, Train Steps/Sec: 0.91 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 45457/45917 [13:13:47<08:30, 1.11s/it][2025-04-25 03:45:12] (step=0091375) Train Loss: 6.3196, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 45482/45917 [13:14:14<07:58, 1.10s/it][2025-04-25 03:45:40] (step=0091400) Train Loss: 6.2833, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 45507/45917 [13:14:42<07:39, 1.12s/it][2025-04-25 03:46:08] (step=0091425) Train Loss: 6.2967, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 45532/45917 [13:15:10<07:05, 1.11s/it][2025-04-25 03:46:35] (step=0091450) Train Loss: 6.2939, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 45557/45917 [13:15:37<06:33, 1.09s/it][2025-04-25 03:47:03] (step=0091475) Train Loss: 6.3149, Train Steps/Sec: 0.90 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 45582/45917 [13:16:05<06:07, 1.10s/it][2025-04-25 03:47:31] (step=0091500) Train Loss: 6.3162, Train Steps/Sec: 0.90 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 45607/45917 [13:16:33<05:47, 1.12s/it][2025-04-25 03:47:59] (step=0091525) Train Loss: 6.2371, Train Steps/Sec: 0.90 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 45632/45917 [13:17:01<05:16, 1.11s/it][2025-04-25 03:48:26] (step=0091550) Train Loss: 6.2834, Train Steps/Sec: 0.90 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 45657/45917 [13:17:28<04:44, 1.09s/it][2025-04-25 03:48:54] (step=0091575) Train Loss: 6.3168, Train Steps/Sec: 0.91 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 45682/45917 [13:17:56<04:16, 1.09s/it][2025-04-25 03:49:22] (step=0091600) Train Loss: 6.2638, Train Steps/Sec: 0.90 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 45707/45917 [13:18:24<03:56, 1.13s/it][2025-04-25 03:49:49] (step=0091625) Train Loss: 6.3145, Train Steps/Sec: 0.90 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 45732/45917 [13:18:51<03:25, 1.11s/it][2025-04-25 03:50:17] (step=0091650) Train Loss: 6.2659, Train Steps/Sec: 0.90 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 45757/45917 [13:19:19<02:57, 1.11s/it][2025-04-25 03:50:45] (step=0091675) Train Loss: 6.3200, Train Steps/Sec: 0.90 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 45782/45917 [13:19:47<02:27, 1.10s/it][2025-04-25 03:51:13] (step=0091700) Train Loss: 6.2762, Train Steps/Sec: 0.90 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 45807/45917 [13:20:15<02:06, 1.15s/it][2025-04-25 03:51:41] (step=0091725) Train Loss: 6.3236, Train Steps/Sec: 0.89 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 45832/45917 [13:20:42<01:35, 1.12s/it][2025-04-25 03:52:08] (step=0091750) Train Loss: 6.3527, Train Steps/Sec: 0.90 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 45857/45917 [13:21:10<01:06, 1.11s/it][2025-04-25 03:52:36] (step=0091775) Train Loss: 6.3383, Train Steps/Sec: 0.90 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 45882/45917 [13:21:38<00:38, 1.11s/it][2025-04-25 03:53:04] (step=0091800) Train Loss: 6.3187, Train Steps/Sec: 0.89 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 45907/45917 [13:22:06<00:11, 1.15s/it][2025-04-25 03:53:32] (step=0091825) Train Loss: 6.3321, Train Steps/Sec: 0.90 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 45917/45917 [13:22:19<00:00, 1.05s/it] +[2025-04-25 03:53:43] Done! diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/requirements.txt b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8d52c600fa38ccbfa2604fb984f24a6dbb7afdb1 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/requirements.txt @@ -0,0 +1,131 @@ +setproctitle==1.3.5 +nvidia-nvjitlink-cu12==12.1.105 +nvidia-nvtx-cu12==12.1.105 +debugpy==1.8.13 +pyarrow==19.0.1 +click==8.1.8 +tqdm==4.67.1 +decorator==5.2.1 +cycler==0.12.1 +pydantic==2.10.6 +jedi==0.19.2 +PyYAML==6.0.2 +torchaudio==2.5.1+cu121 +wheel==0.45.1 +pure_eval==0.2.3 +nvidia-cudnn-cu12==9.1.0.70 +certifi==2025.1.31 +pillow==11.1.0 +typing_extensions==4.12.2 +nvitop==1.4.2 +nvidia-curand-cu12==10.3.2.106 +pyparsing==3.2.1 +docker-pycreds==0.4.0 +frozenlist==1.5.0 +wcwidth==0.2.13 +platformdirs==4.3.6 +jupyter_core==5.7.2 +kiwisolver==1.4.8 +prompt_toolkit==3.0.50 +datasets==3.4.1 +comm==0.2.2 +joblib==1.4.2 +pip==25.0 +transformers==4.49.0 +sentry-sdk==2.23.1 +aiohttp==3.11.14 +nest_asyncio==1.6.0 +triton==3.1.0 +Jinja2==3.1.4 +multiprocess==0.70.16 +fsspec==2024.12.0 +executing==2.2.0 +executing==2.1.0 +nvidia-cuda-cupti-cu12==12.1.105 +attrs==25.3.0 +GitPython==3.1.44 +ptyprocess==0.7.0 +matplotlib-inline==0.1.7 +pydantic_core==2.27.2 +pexpect==4.9.0 +urllib3==2.3.0 +psutil==7.0.0 +jupyter_client==8.6.3 +dill==0.3.8 +wandb==0.19.8 +ipykernel==6.29.5 +aiosignal==1.3.2 +contourpy==1.3.1 +propcache==0.3.0 +networkx==3.3 +setuptools==75.8.0 +nvidia-cusolver-cu12==11.4.5.107 +packaging==24.2 +nltk==3.9.1 +fonttools==4.56.0 +nvidia-ml-py==12.570.86 +smmap==5.0.2 +nvidia-cusparse-cu12==12.1.0.106 +filelock==3.18.0 +nvidia-cublas-cu12==12.1.3.1 +Pygments==2.19.1 +pandas==2.2.3 +torchvision==0.20.1+cu121 +annotated-types==0.7.0 +traitlets==5.14.3 +yarl==1.18.3 +sympy==1.13.1 +pickleshare==0.7.5 +xxhash==3.5.0 +exceptiongroup==1.2.2 +matplotlib==3.10.1 +stack_data==0.6.3 +pytz==2025.1 +protobuf==5.29.3 +nvidia-nccl-cu12==2.21.5 +regex==2024.11.6 +multidict==6.2.0 +MarkupSafe==2.1.5 +gitdb==4.0.12 +nvidia-cufft-cu12==11.0.2.54 +charset-normalizer==3.4.1 +opencv-python==4.11.0.86 +safetensors==0.5.3 +ipython==9.0.2 +asttokens==3.0.0 +ipython_pygments_lexers==1.1.1 +parso==0.8.4 +idna==3.10 +python-dateutil==2.9.0.post0 +tornado==6.4.2 +numpy==2.2.4 +pyzmq==26.3.0 +hf_transfer==0.1.9 +torch==2.5.1+cu121 +nvidia-cuda-nvrtc-cu12==12.1.105 +zipp==3.21.0 +mpmath==1.3.0 +aiohappyeyeballs==2.6.1 +six==1.17.0 +huggingface-hub==0.29.3 +requests==2.32.3 +tzdata==2025.1 +tokenizers==0.21.1 +importlib_metadata==8.6.1 +nvidia-cuda-runtime-cu12==12.1.105 +jaraco.functools==4.0.1 +typeguard==4.3.0 +typing_extensions==4.12.2 +wheel==0.43.0 +platformdirs==4.2.2 +jaraco.context==5.3.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +jaraco.collections==5.1.0 +autocommand==2.2.2 +zipp==3.19.2 +packaging==24.2 +more-itertools==10.3.0 +tomli==2.0.1 +jaraco.text==3.12.1 +inflect==7.3.1 diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/wandb-metadata.json b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d4f0f03f02852d6643a99adef90ff5049a396bfd --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/wandb-metadata.json @@ -0,0 +1,149 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-24T14:31:23.682999Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_trained_raw_cleaned.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects", + "--cloud-save-path", + "/tmp/haozhezhao/MLLMG/checkpoint", + "--lr", + "5e-5", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_val_raw_cleaned.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/instructblip-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "200", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "56", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "4", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "2", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "instructblip", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--train_all", + "--load_fixed_llamagen", + "--save_total_limit", + "1", + "--fix", + "gpt-empty-fix", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0056000.pt", + "--resume" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "git": { + "remote": "git@github.com:HaozheZhao/MLLMG.git", + "commit": "4b66c5ea70beaa476544d6b4715bf5055f7ab018" + }, + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects", + "host": "447cc403a8794092814259713c51c1df00001W", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "59288391680" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/wandb-summary.json b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..304ab61b7bd6a3748ab962eb0d3b6cc016223374 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/files/wandb-summary.json @@ -0,0 +1 @@ +{"train lr":4.570401337191605e-05,"train loss":6.332061767578125,"Train Steps/Sec":0.8976689761411829,"_timestamp":1.745553212405857e+09,"_runtime":48145.611081503,"_step":91825,"_wandb":{"runtime":48145}} \ No newline at end of file diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug-core.log b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..5a1982dc7ddc43c91ea5c4217008f8c1d344fd29 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-04-24T14:31:22.962262217Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp8k6cxysa/port-2575513.txt","pid":2575513,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-24T14:31:22.96392938Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2575513} +{"time":"2025-04-24T14:31:22.963870635Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42271,"Zone":""}} +{"time":"2025-04-24T14:31:23.149022785Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:46114"} +{"time":"2025-04-24T14:31:23.684257955Z","level":"INFO","msg":"handleInformInit: received","streamId":"yx27lyn0","id":"127.0.0.1:46114"} +{"time":"2025-04-24T14:31:23.891684685Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"yx27lyn0","id":"127.0.0.1:46114"} +{"time":"2025-04-25T03:53:49.29390565Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:46114"} +{"time":"2025-04-25T03:53:49.294005287Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:46114"} +{"time":"2025-04-25T03:53:49.294049319Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-04-25T03:53:49.294125392Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:46114"} +{"time":"2025-04-25T03:53:50.692074388Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:46114"} +{"time":"2025-04-25T03:53:50.692108742Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:46114"} +{"time":"2025-04-25T03:53:50.692125523Z","level":"INFO","msg":"server is closed"} diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug-internal.log b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..873f0d256eef5ca67150ae6cdee7899d676d46f1 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-04-24T14:31:23.684611405Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug-core.log"} +{"time":"2025-04-24T14:31:23.891641414Z","level":"INFO","msg":"created new stream","id":"yx27lyn0"} +{"time":"2025-04-24T14:31:23.891678633Z","level":"INFO","msg":"stream: started","id":"yx27lyn0"} +{"time":"2025-04-24T14:31:23.891700099Z","level":"INFO","msg":"writer: Do: started","stream_id":"yx27lyn0"} +{"time":"2025-04-24T14:31:23.891715989Z","level":"INFO","msg":"sender: started","stream_id":"yx27lyn0"} +{"time":"2025-04-24T14:31:23.891910858Z","level":"INFO","msg":"handler: started","stream_id":"yx27lyn0"} +{"time":"2025-04-24T14:31:24.428314594Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-25T03:53:49.294061502Z","level":"INFO","msg":"stream: closing","id":"yx27lyn0"} +{"time":"2025-04-25T03:53:49.294102644Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-25T03:53:49.295055957Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-25T03:53:50.416600201Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-25T03:53:50.691812308Z","level":"INFO","msg":"handler: closed","stream_id":"yx27lyn0"} +{"time":"2025-04-25T03:53:50.691857783Z","level":"INFO","msg":"writer: Close: closed","stream_id":"yx27lyn0"} +{"time":"2025-04-25T03:53:50.691886326Z","level":"INFO","msg":"sender: closed","stream_id":"yx27lyn0"} +{"time":"2025-04-25T03:53:50.691949154Z","level":"INFO","msg":"stream: closed","id":"yx27lyn0"} diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug.log b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..95150326adf7028bd69677201422ae4808d94263 --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug.log @@ -0,0 +1,23 @@ +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_setup.py:_flush():67] Configure stats pid to 2575513 +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug.log +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/logs/debug-internal.log +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_init.py:init():761] calling init triggers +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_trained_raw_cleaned.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/003-GPT-XL/checkpoints/0056000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 5e-05, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_cc12m_gen_val_raw_cleaned.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/instructblip-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 200, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'instructblip', 'do_recovery': True, 'no_replace': False, 'resume': True, 'dreambench_eval': False, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': None, 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-24 14:31:23,676 INFO MainThread:2575513 [wandb_init.py:init():784] starting backend +2025-04-24 14:31:23,677 INFO MainThread:2575513 [wandb_init.py:init():788] sending inform_init request +2025-04-24 14:31:23,682 INFO MainThread:2575513 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-24 14:31:23,682 INFO MainThread:2575513 [wandb_init.py:init():798] backend started and connected +2025-04-24 14:31:23,685 INFO MainThread:2575513 [wandb_init.py:init():891] updated telemetry +2025-04-24 14:31:23,690 INFO MainThread:2575513 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-24 14:31:24,425 INFO MainThread:2575513 [wandb_init.py:init():990] starting run threads in backend +2025-04-24 14:31:24,511 INFO MainThread:2575513 [wandb_run.py:_console_start():2375] atexit reg +2025-04-24 14:31:24,511 INFO MainThread:2575513 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-24 14:31:24,512 INFO MainThread:2575513 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-24 14:31:24,513 INFO MainThread:2575513 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-24 14:31:24,514 INFO MainThread:2575513 [wandb_init.py:init():1032] run started, returning control to user process +2025-04-25 03:53:49,293 INFO MsgRouterThr:2575513 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/run-yx27lyn0.wandb b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/run-yx27lyn0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..70051cf0abe281526131bd3e4015e7b3908048ba --- /dev/null +++ b/EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/run-yx27lyn0.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6b6d7258af7caf837eb4af8aae5642cb5a94274fd84c4eb177af338b603e770 +size 28031219