diff --git a/.gitattributes b/.gitattributes index 0d5b15273fd6271f55981bf5488930b0e91d09bb..bfcd47d1c827b08601e26caea01cd1a1ce21c4e7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -347,3 +347,112 @@ checkpoints/Empyu_Fix_posEmbedding_cls_512_Trainall_llamagen_t2i_stage3_subject_ checkpoints/Empyu_Fix_posEmbedding_cls_512_Trainall_llamagen_t2i_stage3_subject_instructblip-flan-t5-xl_train_subject_t2i_ti2i_120_w_flux_segment/wandb/run-20250319_084200-4rtihoox/run-4rtihoox.wandb filter=lfs diff=lfs merge=lfs -text checkpoints/Empyu_Fix_posEmbedding_cls_512_Trainall_llamagen_t2i_stage3_subject_instructblip-flan-t5-xl_train_subject_t2i_ti2i_120_w_flux_segment/wandb/run-20250319_093106-yg797l0c/run-yg797l0c.wandb filter=lfs diff=lfs merge=lfs -text checkpoints/Empyu_Fix_posEmbedding_cls_512_Trainall_llamagen_t2i_stage3_subject_instructblip-flan-t5-xl_train_subject_t2i_ti2i_120_w_flux_segment/wandb/run-20250319_191050-hvjdgdpz/run-hvjdgdpz.wandb filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/run-mlbv5b7s.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/000-GPT-XL/log.txt b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/000-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..af722f857ea85239d529d228ddf63ce223ff34b2 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/000-GPT-XL/log.txt @@ -0,0 +1,18 @@ +[2025-03-24 07:59:06] Experiment directory created at checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/000-GPT-XL +[2025-03-24 07:59:06] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_dir/subject_ti2i_t2i_stage1_w_flux_segment_mid_700k.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoints', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoints/llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_newfix_train_only_projection_i2i/014-GPT-XL/checkpoints/0022000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=4, lr=0.0005, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=96, global_seed=0, num_workers=4, log_every=25, ckpt_every=4000, gradient_accumulation_steps=1, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_dir/new_1117_validation_set.jsonl_mid_1k.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=512, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference.jsonl', multimodal_encoder='llava', do_recovery=False, no_replace=False, resume=False, dreambench_eval=False, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, i2i=False, rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-03-24 07:59:06] Starting rank=0, seed=0, world_size=8. +[2025-03-24 07:59:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 08:00:12] GPT Parameters: 2,310,680,832 +[2025-03-24 08:00:12] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-03-24 08:00:12] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-03-24 08:00:12] using fused AdamW: True +[2025-03-24 08:00:25] Dataset contains 2,494,674 images +[2025-03-24 08:00:25] Train iters 103944 , warmup 5197.200000000001, len of loader 25986 +[2025-03-24 08:00:39] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/checkpoints/llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_newfix_train_only_projection_i2i/014-GPT-XL/checkpoints/0022000.pt +[2025-03-24 08:00:40] Initial state: steps=0, epochs=0 +[2025-03-24 08:00:40] compiling the model... (may take several minutes) +[2025-03-24 08:00:40] freeze the vit +[2025-03-24 08:00:40] ***** total param is 2310680832 ***** +[2025-03-24 08:00:40] ***** total trained param is 2007501056 ***** +[2025-03-24 08:00:43] Training for 4 epochs... +[2025-03-24 08:00:43] Beginning epoch 0... diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0024000.pt b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0024000.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1093701c3e5f1243c22991eb5a8de75a235efb9 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0024000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56418abdf5e8af6d473331f19a366090a94c384b83739ef442875303634c9838 +size 17322479534 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d91fd6813450eb7e1ce35bb2ab3027a9442bb877 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03f450539dfe4c0bd314ecb3df66142074d84ffbaac87888ec90c5d9b793006f +size 592718 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fe5e3ccb39f1c133368579230b95508e803ce3a5 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39699ac63028b29023de07c32e6ce34e03cf63976c2d3db3f96efd00e750141e +size 752784 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6761ec53100e63f628ae4cc56ff95ca23f4cfdf6 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eef55feed2a610ac51ae98a4704a12fe872d387132d954fa8be8d10747b344b +size 799308 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..012fe781fa52ef56bca4f656da6638600c86606b --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3382acb41d06b462a205b1f70b7ffbda5849511dc45375506da09d9118b000cf +size 554633 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ad2ef730bf41df4de38d12bbc7391dbbd8880f86 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a9e80f3707028b664edf2cb08828e7893d4712413603e16b769c4e09fdd8668 +size 612638 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..390c7abaedb41fa91f4354cd83e0f1814b30d983 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:907095d20108ed4f734225a54d293f8cd4426820308e5d52c20c566aa90c3955 +size 634928 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2f4d3b7b66e13fe4502d84ebcd9291d719de50b8 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c43a0094830bdbae7474193f0654f033d838da55407d9ae222806e7a917d9da8 +size 869050 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a5f4377dbcc4a5b7e43a86cf81ff389ec9d73895 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36f98e25cadaf611e65522a0fc86053dd4bd66d5ab9f8b00100f9a43474956bd +size 655068 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e03d3708dfb4f71bd242bceea2b8c819bd4c5b0e --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_10000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9edb7bda968c52a2b366c1ad23ae2cb313a43de27ee2c7a9e341cc0fdc16dfb +size 688857 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b75fd3b2bbd34e73d2387ca686cd17610015d617 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ba98b1ac136f52bfb549884ea138f18d672c491d4bbf06587468f2ef1c76f54 +size 645554 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9ce8ea3b9ceb1b7077144707d5b3a830987429aa --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a5567cc914405abdfdf3e7ed348961f2c374b62fd03ee46a70cec31c128a35e +size 739063 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..de2e51e3b7c4a5cc3e59356f16cf04af24acde24 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30d0a6b28f76695dfd65526b6ebcafb2fa28f7d10e657b15a75238a9fb236a2a +size 777793 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7426fb05a7e4ebd31b1c2c32c1850c1a99bcbb3d --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:811acb16056eeb79e37e571ecb20fe7977be556f28cbd2a9f90f977fe3cf0929 +size 581751 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5dc14c8dc2829ba7338db173cd091e6a98cf84e7 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d3dd1625e36469e24e08bf13987a65173ab2248e45e9b6f8d95fae8a9b9b65f +size 631973 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2ebaffef5a66a8cf2cb616c30447b7ccef618d33 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:273f46d6d1c44b940c32474694a8060d15248e15c3ed23acdd17a7ae09b9223c +size 638412 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c46fb39a44f4a2e9212f1289c9f5d1789a2240e1 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ddb11609e7c2b7afe3bf31ac7d39dedaacd7c12241e503f5d8bf9a4c9e09ca8 +size 801228 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ed2ef0a9d1ebb40167eaeaf1e10ba503b1f05a65 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e49c9374387609a587169661456394ea0aac53a9ef9d508b8b3ef3cb42c126a +size 646886 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..be12141d019a214b8399b8f06c473109637969ec --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_12000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e1ce3032f3bbbe14af6c659dcab9f68bc8bcedfe5abb979b11a720b5cd63ee4 +size 719038 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a2af521a70becfc48f58a83052e1bcf647855039 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dc7ef53aad96af32aa2945adbbfb3b07c8b6592bc64a9595ab4f9fa7eba58a3 +size 606139 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5680224b8b9efcd28ed8f48f42af771dea0533ae --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd2c4e20a10eb84fad03d649f5414d0b5175cbaf1fec5d03cbc555814863528e +size 737535 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a3d82113b5ebc2bba62900b02c8fb237587ce5d6 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0c382d54ff4087d5e05612dae94810129452b92da64d7c2d3c85069272d48d5 +size 730683 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..abf6cf65d5fd16c01f8cb70257be95592ff67b0d --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00c9baecd23bd586ffd026162758ba1093c3a96977148f4809b2a1088f4eac6a +size 604251 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7f63562fbff11dae2fd065ce2836facf2a286310 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45bc00ee3618d3f9d8aef96904fb75d43f85edf7ac2e584830c93bf3eb28e786 +size 613933 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9f5ec48edef4b0e76fbe19a60b1d61dc68a837f4 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6d11afcdfe757b53c3891933ec67eebfd4cdffecdc2f6df576757b422d2c78d +size 627001 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d7777ed68d89af4b108ee7470abb44b881820db0 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de0bc08bab89a1dd91b8a3ab93b2dec96cf630667a9a8febc0146f513d4a4acd +size 800555 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..135c50aa249f0a4efe7a20af6dc79be51bebd5f5 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73e8d2b9bddf669d9f0fe5414bebe6c830779a1cfcd928dbea801bde268c4dff +size 701821 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e6af26756ae81ab97ae6389cdd203316081e22af --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_14000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:704b24ad4f386f4ae0b07c07c67a5768855ab60e90e63f4b858862b71cf502d0 +size 683603 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..26618b975f75923481b740ff67abe93ab42805f5 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbe51244c4b4caf051db53e9dc349cb24831eb0e48207f92e70df60be156730d +size 612851 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f72f8aa77bd998c774da708e9d5aacf2398572be --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4a9ff0fcef8aa2785ace39b5bd0e7f0567ce52458fc37abfdcd7796f3145823 +size 673035 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..278095ae8824e9252365f0f3f8c1193a91c868ee --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67a566a07a3fa1aa8e99775feeb606f0726a546c016960bbd0df9fb958e78e90 +size 786502 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..65c3fd0b88a9fafacdadf92af419b2cec4a3259a --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b42853a324febd3d7398700241d4af5e40788bd1c90a671399d5c8ee2da14c5 +size 577838 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..93460e084770a650fbec0c5301f0b771c7a8a9fe --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5c1ab1be4413501ac034735598481b24d836e7ec049d904f6ddf62ba113bad0 +size 597455 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..46f59fb8a2717f5f5b734a9790487e95c4270283 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ecd676d51fbb7e605ccbe178fbe4e812276ad218f4a557ec0b67e74bbdecec4 +size 588747 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3c9775e5d84fa02e0b5aa82a28e9a13fc23e3767 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f51ab8ea128113c33d133341af4dfac1b7524a632956ca417d6bd6a65f12511 +size 771066 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bfc7df0ca0e3066152e79ccce207c11e723a27cb --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb7271f40acb4597517e13a01204b62f8bf8501207ec8063ee6eeb051c2f3880 +size 736373 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..31ef1be97a00b564c069604133c440ec2e97a5f2 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_16000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aef9f56c743df18221654e2857b6202b9cd9a2743050388842b8d805553e8b49 +size 753423 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c8ca972aceb63fcb9bc324cfc598aa0625d5c897 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:325b5f51ddb7723272ef75f723fe8c46711f926bcd60158e4421975d5d936c38 +size 646274 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a836e187858f74537d5ccc70e9f353ae09688383 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3729b8ee1f89058cb0ce244420ef3bc25962a285a05b99887ed2f46f1937e27d +size 765065 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ec7f6ee30feb266c5609f3a85ecd15c281e9e166 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44af20b92d78cb0e07ae69dc9a842d4e886cb8cf2571480c6e69cc3b9619c657 +size 761191 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a48752fc87aae9e31b6f2c101543bb65132359b0 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee2de99d886fe9250d10f0324ed89c54f7d0a0a6b55489246721f470405b712c +size 561321 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e4bf2bd1a062f19b471b6d4b738b3e62c795f34b --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0946f8cb59ff3bde220fc41508f4cfbeb1293d9f7b690d70fb944226d751bc3 +size 597558 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ba1a8dd47644e3a486fa4f6014850413dccc94e0 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc1aac1b887e433a882ee5ba6a64948632cc05fc90d7a8402fa88881f8beb244 +size 646224 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..13ebb987be75e8eb4303ec7bf331142932959a61 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2856d800f6b5663739e46ed75de072442473224d8779df89093682381d930519 +size 845972 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bfb0705faf49c04ec79fd1f3cd126aad8f42c58b --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7124182ed43d6f37e4c8f825c7e102c3007c7d444ec1e31b440346aa105f1b5b +size 690058 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..20fcc9a8b21e5200da68abd9d2858fa18fb62915 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_18000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fecc5168b7593abbf47f5d07ef9d8b3b126125b6ccd1e3a813002b316ce7c573 +size 687732 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9020b930eba6050eca483d87699b7cecf72d8941 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bbf587a3fb8afe705052d7d82ffdc80566815e30ea6576b043b87760acf7211 +size 703123 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0d5d6ddf49a7a32e956c8291c49fb167ca8b2424 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c74d4497f20acb8cc8aefe6519b1d0b3fdd3123af12fede4ff0450118a63940a +size 683156 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..adc144be2ca80fd79f05e60acda002e03ebf7ac2 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44ba9512f7f06acdb00568f1b8957aea6089e51be41492be6c3f9cb7dee68326 +size 863394 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5e17d9e468fbc0cd6487efe9be7a94b9f920d750 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef07fe2d708a49f1ec1f6378cea2f301830ca9767d20f68e425d1e8f10db1483 +size 604521 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c3617488ff49bbebb3ea6260a33f2769ae79f1b6 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54b2aa36c1a1bade8949723187367ef4cc9e3829f974283254899a9f9a2084e8 +size 622951 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4be405d65fdb040e9af87db1a0e27bf08a304cda --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6858b5e905ba86652d821909f7408c5f797857de57c5ab8e4b9f62f521d817d +size 622120 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..26e96b2ae7ef1c190906577a2b7e60df8a094244 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f4b7e2a9988c24b00ca2227affafbb6dea1a456199b39e81fc28340c00509fb +size 883826 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f7c6e362220d513fdd6f76c836d7d72848ffe7ca --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db74634a27df19563cdaa3905f9782e301fbcfb0173550754d202150dc4822a8 +size 650392 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..16e8f726ad76146675c52f11b3ea390566dbf4f9 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_2000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fa8934c87da1748ef87d7f241b6971b10e94784d78be3520f134f9c257930b7 +size 713562 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..89d5f8b5726c3c2dcb1909bf2d1a8619b9c9fa3b --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bd0ec73362724d8aa3d39148f542d979ce65ab39bb853b2756c4ffe7e571a52 +size 646485 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3c835d56bea90baa7e308099e85d075388c42651 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:298f734c3d0a37808a0fc1f25b8580918d71b0e619e053fa192a2f518e0c13a8 +size 741103 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4be17c6c3ed5fe4c71376ce5f2b7ef20a9c22565 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b56132ce91505da1398198ce893122832c402fac91ae78942b90e6f42c6f1003 +size 791855 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f2962a4b1e1f4ddca021926a7712899b903cb7b7 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edcf1209ce2f60076718df95ec5fe67a6e329319ec08d6160d0501c9ba33cd51 +size 585935 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..45ae307f3028c6c97826a6bea5e301024cabcbf5 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:744ac443ff03bc93f4cbee5c44f27355217fa2b7afd5376ad17c356a4fd9598a +size 605763 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..730206053dc13e9a357f45318b3e36a50e1e7602 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4367082b33e61217fa9766575e40893901f8743cf8d0e423e2f1d195e19bb84 +size 664634 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4d80edd41eb8926136d4ef69c489cee92e221c10 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48aaf62fbeb91470393945ab804b9c5bc6a5b886dcd238e08920d295be5e1917 +size 802275 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0761e6f7cad27f1894e7b9c312e2cbd3cb42e5f6 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7c6c374b7fa34899c3efa8a781acd1399ad3d6db5889aefffdb1039d3c50b22 +size 675409 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..187049e43615fbaa6e4defdf30c52d289a1785d6 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_20000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:445881acda253f83146e2ce6ff78b7890f399037bdf85eab4d5bb062baff1d97 +size 706412 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..39151fc2633ea79dc02de484331128fd68c0cff1 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79a7cc134a3c0b5971c98faae2f5e7f175f73a3538a15d13446946248e709031 +size 596721 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..525ff975abd07bcc00967d76c8d361a31cffc7b1 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d31f177fe3e9507c270ee4d502a089698dc8eb9f563683a53cb2d5adc4dfface +size 739680 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..07c93134a8d839f420f7d842b7ef77bfafd18719 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:125fdeba7062e79dc0a8a65ad91b69a3b1eac7eb8c2e01c17792235929fdb224 +size 758489 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..00a278797e1cf4ce3aa541c436cf4425f3e91d3e --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36b336b4bfe59e82d2d4931020923443653403316047fd72aad23b928291cedc +size 554020 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0774f0c731e5b734e73a689445b4da3e0784da90 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48847566a15e51a2d4ad37ca52de45aa49f6fa4f845bb59e8c79124be12d9cac +size 581869 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cb28d198e9c3c620e5a81a97deb46b83227e9a10 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6decb2bac2c86b8d48f8f900fac6df839962d8a88cf327c60c9f80ea27055d28 +size 582163 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..019c01001c9cb4a4913f0c4ca909331d225ef4e6 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02185ecd08ed35ca4813a0e871c8bb601ce2e081fa528b7604890da702dc5848 +size 802648 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2a811178d56926c3be92ae35b9d964105c44c87f --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8076efeb23b61fbca50da04d3ded2b5b7c37055504913fb1e132c6a70c4c784b +size 669904 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..977d82be62c9d3de27f4a29e32a7f126d6ae5290 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_22000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff152b9b731c1266f139e10f175e802674d9b11ec29de72950e29179f3f1a344 +size 675309 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b4e4326315d0c7924cee4086844670a373fa7648 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:596b433e475b29761c58df59a958b1bd23b8f5e3cce8afaa9a3b694a269a0c6c +size 591046 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6f205baef95def8729ff39e232adca7bb8c2a2f7 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6011ada2014d2e3fab129edfefdcd409b37368d8ab5a29700fa007624fc7015b +size 683469 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e136c5da9acdf0ae5094a10db29230c618daa69e --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df0b8d98baad59934a8f8de57b75ba8be9b30bb752ebb7417af62e93f6a4acc4 +size 770795 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..274c9603ac7eae276e6ecef52cac56f45b61b3d9 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0978a47d732afa8df0262f400eb843b23dc71c52e1a1f5d2e353f67f53e67198 +size 552634 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..934a005b9b9f855901ab6eec74f64ca16109906c --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d388751dc8399ac61cc36eef4d743236cc268c064594a80e3b2d4624d3b844b4 +size 603062 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fb92aed03183241819ae42777feeb5f479923220 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4378926ce208604d9fc48a0ec52d4fd153563e7cd0947fa0780b42aada8265d9 +size 604640 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9db61173e4a00b79dffcc47d1aec37ecaacd3788 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8afc4b6cfb9df5434aa2e493643f24c86e9ac96b2e5fb5838cf7c4a4687e958 +size 798955 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..81b3add42af883f9c04f4b1ef9887b837010e429 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac8fa3f0b30948a96aed9d19e7c49780aea56722921d2c8a8b822cf5303f6f36 +size 684710 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2f254347650f45392787d8c81ac89e6d2231261b --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_24000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba718c4ab96e4dea95018a771741fb280f37d2bfe54101f7e0a209299e4503c0 +size 704404 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d08ed195f22f1e3eeab5f919a6378aea92411015 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0e548a94bac05e3bb6adb23260debfc461a4f622941f21adf224f817faca803 +size 671856 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c94b80788eaceca59ca4ffad1684f0735209e2fe --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcccdbcd12e177cb06f8e417fd71a5f43f132620e675abce123e2d9bfedf136d +size 757260 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5b05ea3f56d7dc678ac8d8037325d2315da7f2f6 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c387aff94ec6be4037a0e98d019cf459b6041c90301876f7f9d488f0866cdb5f +size 837150 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..446c1d68192037ddcda795d632151e05f087758f --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8419156fd650e324c4e087b2620e5df7cf3642a458946bf7b50008dfac5c985c +size 635776 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0da3dbc0bb3c1138eb9878f840fc5e82cf207abb --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e3a5d019037cc7907a25e96e0ffa155c40c1dd0b50f19eda4edaccdeb0f0ea2 +size 667685 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8c120441c9a212c2eabac1bac5e12a2dae493730 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bbdc3284d35779911e22a589d93aa2640429f1c1e59f4a8d6abc68c943c0b2c +size 668639 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c192b84f5c0a2ba0e3fe84489b898f7367b7a692 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6d34f7558f892c2fa7b3a5745482d4095a9ab3c5264d05aed5c929b11525a5e +size 838890 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ca58181c3d0077fce54ec32f87d25587626a5d8d --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cb4817ca03d087d1e42595b30a101ece2d253c68ce0a0ca66c836c9a511fc89 +size 685841 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9b23767e2e73970b28108509397a350c4f236fbe --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_4000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0e39f01eab4e5e7b6eec67ce982254cd3ab8c7b7a36830bd1adb9c3114dfd82 +size 782691 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..96c65eb3b2c55e78006dc38a4d56aff22fb4d3cf --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3cafe9cb043205243eece55079896b2e875d452aba7e412dafcb22bc3c61a49 +size 608288 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ee49f1958f0f329ac0ef6156bc700f3932b0cbde --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9428637c457bc7b68873978d59732c1589d4f5d9518645ada8e2b86957bdd9d +size 734674 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..009d47e1ba6ab112db59f4853ff6feddc198d792 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a70d0f8f64aa494628646d31752fe59a7130e1130bb4bef4e58bc12dc089c4e0 +size 823502 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6f1019c30aece747a6ebbaa87e62136fe61fb83c --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f3106287dc39bce249052ed4a9f8055a788fdc61a79c0fd7a65ef27147ca120 +size 621014 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ef439b0057e175373f4eec5758387a089ee9e3a5 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:490bfe6a6bec7e57514eefc4242b2eda7f2cd963aafca42cf6fb34e04b249ffb +size 619756 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..341a799f8e5605123f9bd696ee01009670764c9e --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f97af14642801798e23d51b42d916f9b23f4c4b05a97d1844183114c9755e42 +size 646060 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4c1f4b930f04c5d374391bb818600fe3de9ebd02 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9a02f96d41916925a0b29a85a4786d6882f52b91f5024806e1f041d038bf005 +size 820268 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bada3fb33fbaccadc970f55865fe6712b0ce0505 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46d4348dfca46aa2f71bb440e5ee91a6d02ba558d8e12a44649ffb4abd6074d3 +size 625596 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9b2324723b4dd8ea1813662e08090dd0733240e1 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_6000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c595750f28c31ea6417ba0b3ccf42816cf520d16a80c91007d04dca681665c3b +size 755238 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5f173f013aabffa4eb825832fd5315190615c7da --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c5db6bb2922466f3426827c79aaee25c30659f7491581c84facb5cbe565fd75 +size 594875 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a72395a7bdc873cb02f65820585ba94f7f9bbf00 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6588504977ec61837765cf3284d37d134b12effbc4c73095d9956d1b8902824 +size 743788 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..df4f3f4281e024803cf56675d45a472d0b9f5deb --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:937bbac345990ee08523e4305698e65c61d358ce4e36527bb1f1f31fbef7f065 +size 788955 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..96fb3171baaa153c7cccf94e6dfbb8e35cb56781 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bfb131a5a3286d8992134648be0e58cfda8a415e3236b3b81c40f41c6dca487 +size 622329 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5708caaa25eb86494baf770cef9b219ecf749d26 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:542606028d8d6ec98dc64db60da3cd01e2c120bceb995f5d5347303d8285455e +size 579434 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_5_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7ba8d2e78904e7e366d1aece43fb14c78000d251 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a4fed75e5c9803f357104c6facab06342a127f1d8f5893c15e273f8400a8f75 +size 631431 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_6_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c2615047c09667db9df780c563d7840fd57a342d --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7eea6d100ae8d33384747ddc4705f5569af32064cfe45dee06827958298ec3ed +size 831060 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_7_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a2cbcacdde33c0a5bd978e6e5f1009df890f58ec --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bc58da100922a846cbe6d3f3bfeb61204b222fadb9a38fa7f7d3209f277d7a7 +size 676808 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_8_cfg_7.5_topk_16384.jpg b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..86560327464d439534a8de80c13335fd0f908a53 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/eval_step_8000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc759ab84977d4f71b808ac79115bdea21156d6a521bb1abc201a659957d9650 +size 728539 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/log.txt b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..efa3f3e5398844f6e0fd107196dede61b94a812a --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/log.txt @@ -0,0 +1,1081 @@ +[2025-03-24 08:27:04] Experiment directory created at checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL +[2025-03-24 08:27:04] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_dir/subject_ti2i_t2i_stage1_w_flux_segment_mid_700k.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoints', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoints/llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_newfix_train_only_projection_i2i/014-GPT-XL/checkpoints/0022000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=4, lr=0.0005, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=4, log_every=25, ckpt_every=4000, gradient_accumulation_steps=1, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_dir/new_1117_validation_set.jsonl_mid_1k.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=512, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference.jsonl', multimodal_encoder='llava', do_recovery=False, no_replace=False, resume=False, dreambench_eval=False, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, i2i=False, rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-03-24 08:27:04] Starting rank=0, seed=0, world_size=8. +[2025-03-24 08:27:04] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 08:28:10] GPT Parameters: 2,310,680,832 +[2025-03-24 08:28:10] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-03-24 08:28:10] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-03-24 08:28:10] using fused AdamW: True +[2025-03-24 08:28:24] Dataset contains 2,494,674 images +[2025-03-24 08:28:24] Train iters 178188 , warmup 8909.4, len of loader 44547 +[2025-03-24 08:28:36] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/checkpoints/llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_newfix_train_only_projection_i2i/014-GPT-XL/checkpoints/0022000.pt +[2025-03-24 08:28:36] Initial state: steps=0, epochs=0 +[2025-03-24 08:28:36] compiling the model... (may take several minutes) +[2025-03-24 08:28:37] freeze the vit +[2025-03-24 08:28:37] ***** total param is 2310680832 ***** +[2025-03-24 08:28:37] ***** total trained param is 2007501056 ***** +[2025-03-24 08:28:41] Training for 4 epochs... +[2025-03-24 08:28:41] Beginning epoch 0... +[2025-03-24 08:32:46] (step=0000025) Train Loss: 3.9959, Train Steps/Sec: 0.10 +[2025-03-24 08:33:09] (step=0000050) Train Loss: 4.0003, Train Steps/Sec: 1.10 +[2025-03-24 08:33:32] (step=0000075) Train Loss: 3.9658, Train Steps/Sec: 1.10 +[2025-03-24 08:33:55] (step=0000100) Train Loss: 4.0230, Train Steps/Sec: 1.10 +[2025-03-24 08:34:18] (step=0000125) Train Loss: 3.9988, Train Steps/Sec: 1.10 +[2025-03-24 08:34:40] (step=0000150) Train Loss: 4.0590, Train Steps/Sec: 1.10 +[2025-03-24 08:35:03] (step=0000175) Train Loss: 3.9180, Train Steps/Sec: 1.09 +[2025-03-24 08:35:26] (step=0000200) Train Loss: 4.0239, Train Steps/Sec: 1.10 +[2025-03-24 08:35:50] (step=0000225) Train Loss: 4.0382, Train Steps/Sec: 1.03 +[2025-03-24 08:36:16] (step=0000250) Train Loss: 3.9438, Train Steps/Sec: 0.99 +[2025-03-24 08:36:39] (step=0000275) Train Loss: 4.0125, Train Steps/Sec: 1.08 +[2025-03-24 08:37:02] (step=0000300) Train Loss: 3.9160, Train Steps/Sec: 1.06 +[2025-03-24 08:37:58] (step=0000325) Train Loss: 3.9662, Train Steps/Sec: 0.45 +[2025-03-24 08:38:23] (step=0000350) Train Loss: 3.9897, Train Steps/Sec: 1.03 +[2025-03-24 08:39:21] (step=0000375) Train Loss: 3.9058, Train Steps/Sec: 0.43 +[2025-03-24 08:39:44] (step=0000400) Train Loss: 3.8840, Train Steps/Sec: 1.10 +[2025-03-24 08:40:07] (step=0000425) Train Loss: 3.9864, Train Steps/Sec: 1.10 +[2025-03-24 08:40:30] (step=0000450) Train Loss: 3.8547, Train Steps/Sec: 1.05 +[2025-03-24 08:40:53] (step=0000475) Train Loss: 4.0110, Train Steps/Sec: 1.10 +[2025-03-24 08:41:17] (step=0000500) Train Loss: 4.0012, Train Steps/Sec: 1.06 +[2025-03-24 08:41:40] (step=0000525) Train Loss: 3.9500, Train Steps/Sec: 1.10 +[2025-03-24 08:42:29] (step=0000550) Train Loss: 3.8852, Train Steps/Sec: 0.51 +[2025-03-24 08:42:52] (step=0000575) Train Loss: 3.9429, Train Steps/Sec: 1.10 +[2025-03-24 08:43:16] (step=0000600) Train Loss: 3.9124, Train Steps/Sec: 1.04 +[2025-03-24 08:43:38] (step=0000625) Train Loss: 3.9477, Train Steps/Sec: 1.09 +[2025-03-24 08:44:01] (step=0000650) Train Loss: 3.9705, Train Steps/Sec: 1.10 +[2025-03-24 08:44:25] (step=0000675) Train Loss: 3.8690, Train Steps/Sec: 1.06 +[2025-03-24 08:44:48] (step=0000700) Train Loss: 3.9458, Train Steps/Sec: 1.10 +[2025-03-24 08:45:11] (step=0000725) Train Loss: 3.8558, Train Steps/Sec: 1.10 +[2025-03-24 08:46:07] (step=0000750) Train Loss: 4.0343, Train Steps/Sec: 0.44 +[2025-03-24 08:46:30] (step=0000775) Train Loss: 3.9443, Train Steps/Sec: 1.10 +[2025-03-24 08:46:53] (step=0000800) Train Loss: 3.8730, Train Steps/Sec: 1.10 +[2025-03-24 08:47:17] (step=0000825) Train Loss: 3.8826, Train Steps/Sec: 1.05 +[2025-03-24 08:47:40] (step=0000850) Train Loss: 3.9439, Train Steps/Sec: 1.06 +[2025-03-24 08:48:04] (step=0000875) Train Loss: 3.9147, Train Steps/Sec: 1.06 +[2025-03-24 08:48:28] (step=0000900) Train Loss: 3.9774, Train Steps/Sec: 1.01 +[2025-03-24 08:48:51] (step=0000925) Train Loss: 3.8880, Train Steps/Sec: 1.10 +[2025-03-24 08:49:15] (step=0000950) Train Loss: 3.9122, Train Steps/Sec: 1.06 +[2025-03-24 08:49:38] (step=0000975) Train Loss: 3.8948, Train Steps/Sec: 1.10 +[2025-03-24 08:50:00] (step=0001000) Train Loss: 3.8736, Train Steps/Sec: 1.10 +[2025-03-24 08:50:23] (step=0001025) Train Loss: 3.8799, Train Steps/Sec: 1.10 +[2025-03-24 08:50:46] (step=0001050) Train Loss: 3.8845, Train Steps/Sec: 1.10 +[2025-03-24 08:51:09] (step=0001075) Train Loss: 3.9695, Train Steps/Sec: 1.10 +[2025-03-24 08:51:31] (step=0001100) Train Loss: 3.9026, Train Steps/Sec: 1.10 +[2025-03-24 08:51:54] (step=0001125) Train Loss: 3.9809, Train Steps/Sec: 1.10 +[2025-03-24 08:52:17] (step=0001150) Train Loss: 3.9817, Train Steps/Sec: 1.10 +[2025-03-24 08:52:40] (step=0001175) Train Loss: 3.9306, Train Steps/Sec: 1.10 +[2025-03-24 08:53:02] (step=0001200) Train Loss: 3.9307, Train Steps/Sec: 1.10 +[2025-03-24 08:53:26] (step=0001225) Train Loss: 3.9687, Train Steps/Sec: 1.06 +[2025-03-24 08:53:49] (step=0001250) Train Loss: 3.8205, Train Steps/Sec: 1.07 +[2025-03-24 08:54:13] (step=0001275) Train Loss: 3.8398, Train Steps/Sec: 1.05 +[2025-03-24 08:54:36] (step=0001300) Train Loss: 3.9909, Train Steps/Sec: 1.10 +[2025-03-24 08:55:00] (step=0001325) Train Loss: 3.9139, Train Steps/Sec: 1.06 +[2025-03-24 08:55:56] (step=0001350) Train Loss: 3.8274, Train Steps/Sec: 0.44 +[2025-03-24 08:56:20] (step=0001375) Train Loss: 3.9361, Train Steps/Sec: 1.04 +[2025-03-24 08:56:43] (step=0001400) Train Loss: 3.9242, Train Steps/Sec: 1.10 +[2025-03-24 08:57:05] (step=0001425) Train Loss: 3.8405, Train Steps/Sec: 1.10 +[2025-03-24 08:58:03] (step=0001450) Train Loss: 4.0001, Train Steps/Sec: 0.44 +[2025-03-24 08:58:27] (step=0001475) Train Loss: 3.9757, Train Steps/Sec: 1.05 +[2025-03-24 08:58:49] (step=0001500) Train Loss: 3.8700, Train Steps/Sec: 1.10 +[2025-03-24 08:59:12] (step=0001525) Train Loss: 3.8754, Train Steps/Sec: 1.10 +[2025-03-24 08:59:35] (step=0001550) Train Loss: 3.8921, Train Steps/Sec: 1.10 +[2025-03-24 08:59:58] (step=0001575) Train Loss: 3.9929, Train Steps/Sec: 1.10 +[2025-03-24 09:00:20] (step=0001600) Train Loss: 3.8603, Train Steps/Sec: 1.10 +[2025-03-24 09:00:44] (step=0001625) Train Loss: 3.9080, Train Steps/Sec: 1.06 +[2025-03-24 09:01:07] (step=0001650) Train Loss: 3.9540, Train Steps/Sec: 1.10 +[2025-03-24 09:01:29] (step=0001675) Train Loss: 3.9920, Train Steps/Sec: 1.10 +[2025-03-24 09:01:52] (step=0001700) Train Loss: 3.9888, Train Steps/Sec: 1.10 +[2025-03-24 09:02:15] (step=0001725) Train Loss: 3.9193, Train Steps/Sec: 1.10 +[2025-03-24 09:02:38] (step=0001750) Train Loss: 3.9614, Train Steps/Sec: 1.10 +[2025-03-24 09:03:00] (step=0001775) Train Loss: 3.8379, Train Steps/Sec: 1.10 +[2025-03-24 09:03:23] (step=0001800) Train Loss: 3.8329, Train Steps/Sec: 1.10 +[2025-03-24 09:03:46] (step=0001825) Train Loss: 3.8130, Train Steps/Sec: 1.10 +[2025-03-24 09:04:09] (step=0001850) Train Loss: 3.9063, Train Steps/Sec: 1.06 +[2025-03-24 09:04:33] (step=0001875) Train Loss: 3.9991, Train Steps/Sec: 1.06 +[2025-03-24 09:04:57] (step=0001900) Train Loss: 3.8894, Train Steps/Sec: 1.06 +[2025-03-24 09:05:20] (step=0001925) Train Loss: 3.9092, Train Steps/Sec: 1.06 +[2025-03-24 09:05:44] (step=0001950) Train Loss: 3.9414, Train Steps/Sec: 1.05 +[2025-03-24 09:06:07] (step=0001975) Train Loss: 3.8867, Train Steps/Sec: 1.10 +[2025-03-24 09:06:30] (step=0002000) Train Loss: 3.9432, Train Steps/Sec: 1.10 +[2025-03-24 09:06:30] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 09:15:32] Finish Eval in 2000 steps... +[2025-03-24 09:15:55] (step=0002025) Train Loss: 3.9700, Train Steps/Sec: 0.04 +[2025-03-24 09:16:18] (step=0002050) Train Loss: 3.9817, Train Steps/Sec: 1.09 +[2025-03-24 09:16:41] (step=0002075) Train Loss: 4.0609, Train Steps/Sec: 1.10 +[2025-03-24 09:17:04] (step=0002100) Train Loss: 3.9451, Train Steps/Sec: 1.10 +[2025-03-24 09:17:27] (step=0002125) Train Loss: 3.8427, Train Steps/Sec: 1.10 +[2025-03-24 09:17:50] (step=0002150) Train Loss: 3.8953, Train Steps/Sec: 1.06 +[2025-03-24 09:18:13] (step=0002175) Train Loss: 3.8923, Train Steps/Sec: 1.10 +[2025-03-24 09:18:36] (step=0002200) Train Loss: 3.9512, Train Steps/Sec: 1.10 +[2025-03-24 09:18:59] (step=0002225) Train Loss: 3.8496, Train Steps/Sec: 1.10 +[2025-03-24 09:19:21] (step=0002250) Train Loss: 3.9492, Train Steps/Sec: 1.10 +[2025-03-24 09:19:45] (step=0002275) Train Loss: 3.9039, Train Steps/Sec: 1.06 +[2025-03-24 09:20:09] (step=0002300) Train Loss: 3.8993, Train Steps/Sec: 1.05 +[2025-03-24 09:20:31] (step=0002325) Train Loss: 3.9844, Train Steps/Sec: 1.10 +[2025-03-24 09:20:54] (step=0002350) Train Loss: 3.9886, Train Steps/Sec: 1.10 +[2025-03-24 09:21:17] (step=0002375) Train Loss: 3.9455, Train Steps/Sec: 1.10 +[2025-03-24 09:21:41] (step=0002400) Train Loss: 3.9708, Train Steps/Sec: 1.06 +[2025-03-24 09:22:03] (step=0002425) Train Loss: 3.9303, Train Steps/Sec: 1.10 +[2025-03-24 09:22:26] (step=0002450) Train Loss: 3.9366, Train Steps/Sec: 1.10 +[2025-03-24 09:22:49] (step=0002475) Train Loss: 3.8861, Train Steps/Sec: 1.10 +[2025-03-24 09:23:12] (step=0002500) Train Loss: 3.9588, Train Steps/Sec: 1.10 +[2025-03-24 09:23:35] (step=0002525) Train Loss: 3.7952, Train Steps/Sec: 1.06 +[2025-03-24 09:23:59] (step=0002550) Train Loss: 3.8819, Train Steps/Sec: 1.05 +[2025-03-24 09:24:23] (step=0002575) Train Loss: 3.9877, Train Steps/Sec: 1.06 +[2025-03-24 09:24:45] (step=0002600) Train Loss: 4.0147, Train Steps/Sec: 1.10 +[2025-03-24 09:25:08] (step=0002625) Train Loss: 3.7912, Train Steps/Sec: 1.10 +[2025-03-24 09:25:31] (step=0002650) Train Loss: 3.9630, Train Steps/Sec: 1.10 +[2025-03-24 09:25:54] (step=0002675) Train Loss: 3.8015, Train Steps/Sec: 1.10 +[2025-03-24 09:26:16] (step=0002700) Train Loss: 4.0650, Train Steps/Sec: 1.10 +[2025-03-24 09:27:12] (step=0002725) Train Loss: 3.9187, Train Steps/Sec: 0.45 +[2025-03-24 09:27:35] (step=0002750) Train Loss: 3.9737, Train Steps/Sec: 1.10 +[2025-03-24 09:27:58] (step=0002775) Train Loss: 3.9534, Train Steps/Sec: 1.10 +[2025-03-24 09:28:21] (step=0002800) Train Loss: 3.9969, Train Steps/Sec: 1.06 +[2025-03-24 09:28:45] (step=0002825) Train Loss: 4.0330, Train Steps/Sec: 1.05 +[2025-03-24 09:29:09] (step=0002850) Train Loss: 3.9133, Train Steps/Sec: 1.05 +[2025-03-24 09:29:33] (step=0002875) Train Loss: 4.0268, Train Steps/Sec: 1.05 +[2025-03-24 09:29:57] (step=0002900) Train Loss: 3.9411, Train Steps/Sec: 1.06 +[2025-03-24 09:30:19] (step=0002925) Train Loss: 3.9871, Train Steps/Sec: 1.10 +[2025-03-24 09:30:43] (step=0002950) Train Loss: 3.9881, Train Steps/Sec: 1.06 +[2025-03-24 09:31:06] (step=0002975) Train Loss: 3.8819, Train Steps/Sec: 1.10 +[2025-03-24 09:31:28] (step=0003000) Train Loss: 3.9259, Train Steps/Sec: 1.10 +[2025-03-24 09:31:51] (step=0003025) Train Loss: 3.9605, Train Steps/Sec: 1.10 +[2025-03-24 09:32:14] (step=0003050) Train Loss: 3.9207, Train Steps/Sec: 1.10 +[2025-03-24 09:32:37] (step=0003075) Train Loss: 3.9417, Train Steps/Sec: 1.10 +[2025-03-24 09:32:59] (step=0003100) Train Loss: 3.9905, Train Steps/Sec: 1.10 +[2025-03-24 09:33:22] (step=0003125) Train Loss: 3.9499, Train Steps/Sec: 1.10 +[2025-03-24 09:33:45] (step=0003150) Train Loss: 3.9086, Train Steps/Sec: 1.10 +[2025-03-24 09:34:08] (step=0003175) Train Loss: 3.8718, Train Steps/Sec: 1.10 +[2025-03-24 09:34:31] (step=0003200) Train Loss: 3.9283, Train Steps/Sec: 1.10 +[2025-03-24 09:34:55] (step=0003225) Train Loss: 3.8724, Train Steps/Sec: 1.02 +[2025-03-24 09:35:19] (step=0003250) Train Loss: 3.8557, Train Steps/Sec: 1.05 +[2025-03-24 09:35:42] (step=0003275) Train Loss: 3.9123, Train Steps/Sec: 1.10 +[2025-03-24 09:36:04] (step=0003300) Train Loss: 3.8896, Train Steps/Sec: 1.10 +[2025-03-24 09:36:27] (step=0003325) Train Loss: 3.9486, Train Steps/Sec: 1.10 +[2025-03-24 09:36:50] (step=0003350) Train Loss: 3.8831, Train Steps/Sec: 1.10 +[2025-03-24 09:37:40] (step=0003375) Train Loss: 3.9912, Train Steps/Sec: 0.50 +[2025-03-24 09:38:03] (step=0003400) Train Loss: 3.9816, Train Steps/Sec: 1.10 +[2025-03-24 09:38:26] (step=0003425) Train Loss: 3.8208, Train Steps/Sec: 1.10 +[2025-03-24 09:38:49] (step=0003450) Train Loss: 4.0445, Train Steps/Sec: 1.10 +[2025-03-24 09:39:12] (step=0003475) Train Loss: 3.9729, Train Steps/Sec: 1.06 +[2025-03-24 09:39:36] (step=0003500) Train Loss: 3.8777, Train Steps/Sec: 1.05 +[2025-03-24 09:39:59] (step=0003525) Train Loss: 3.8322, Train Steps/Sec: 1.10 +[2025-03-24 09:40:21] (step=0003550) Train Loss: 3.9794, Train Steps/Sec: 1.10 +[2025-03-24 09:40:44] (step=0003575) Train Loss: 3.8691, Train Steps/Sec: 1.10 +[2025-03-24 09:41:07] (step=0003600) Train Loss: 4.0630, Train Steps/Sec: 1.10 +[2025-03-24 09:41:30] (step=0003625) Train Loss: 3.8853, Train Steps/Sec: 1.10 +[2025-03-24 09:42:21] (step=0003650) Train Loss: 3.9747, Train Steps/Sec: 0.49 +[2025-03-24 09:42:45] (step=0003675) Train Loss: 3.8727, Train Steps/Sec: 1.04 +[2025-03-24 09:43:08] (step=0003700) Train Loss: 4.0063, Train Steps/Sec: 1.10 +[2025-03-24 09:43:31] (step=0003725) Train Loss: 4.0639, Train Steps/Sec: 1.10 +[2025-03-24 09:43:54] (step=0003750) Train Loss: 4.0355, Train Steps/Sec: 1.10 +[2025-03-24 09:44:16] (step=0003775) Train Loss: 4.0620, Train Steps/Sec: 1.10 +[2025-03-24 09:44:40] (step=0003800) Train Loss: 3.9178, Train Steps/Sec: 1.05 +[2025-03-24 09:45:03] (step=0003825) Train Loss: 3.8701, Train Steps/Sec: 1.10 +[2025-03-24 09:45:26] (step=0003850) Train Loss: 3.8323, Train Steps/Sec: 1.10 +[2025-03-24 09:45:50] (step=0003875) Train Loss: 4.0656, Train Steps/Sec: 1.05 +[2025-03-24 09:46:12] (step=0003900) Train Loss: 3.9686, Train Steps/Sec: 1.10 +[2025-03-24 09:46:35] (step=0003925) Train Loss: 3.9444, Train Steps/Sec: 1.10 +[2025-03-24 09:46:58] (step=0003950) Train Loss: 4.0640, Train Steps/Sec: 1.10 +[2025-03-24 09:47:21] (step=0003975) Train Loss: 3.9303, Train Steps/Sec: 1.10 +[2025-03-24 09:47:43] (step=0004000) Train Loss: 3.9198, Train Steps/Sec: 1.10 +[2025-03-24 09:47:43] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 09:56:47] Finish Eval in 4000 steps... +[2025-03-24 09:57:05] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0004000.pt +[2025-03-24 09:57:29] (step=0004025) Train Loss: 3.9170, Train Steps/Sec: 0.04 +[2025-03-24 09:57:52] (step=0004050) Train Loss: 3.9599, Train Steps/Sec: 1.10 +[2025-03-24 09:58:14] (step=0004075) Train Loss: 4.1099, Train Steps/Sec: 1.10 +[2025-03-24 09:58:38] (step=0004100) Train Loss: 4.0088, Train Steps/Sec: 1.05 +[2025-03-24 09:59:02] (step=0004125) Train Loss: 3.9768, Train Steps/Sec: 1.06 +[2025-03-24 09:59:25] (step=0004150) Train Loss: 3.9963, Train Steps/Sec: 1.06 +[2025-03-24 09:59:48] (step=0004175) Train Loss: 3.9638, Train Steps/Sec: 1.10 +[2025-03-24 10:00:11] (step=0004200) Train Loss: 3.9720, Train Steps/Sec: 1.10 +[2025-03-24 10:00:33] (step=0004225) Train Loss: 3.9144, Train Steps/Sec: 1.10 +[2025-03-24 10:00:56] (step=0004250) Train Loss: 3.9741, Train Steps/Sec: 1.10 +[2025-03-24 10:01:19] (step=0004275) Train Loss: 3.9080, Train Steps/Sec: 1.10 +[2025-03-24 10:01:42] (step=0004300) Train Loss: 3.9658, Train Steps/Sec: 1.10 +[2025-03-24 10:02:06] (step=0004325) Train Loss: 3.9719, Train Steps/Sec: 1.01 +[2025-03-24 10:02:29] (step=0004350) Train Loss: 3.9709, Train Steps/Sec: 1.10 +[2025-03-24 10:02:52] (step=0004375) Train Loss: 3.9957, Train Steps/Sec: 1.10 +[2025-03-24 10:03:15] (step=0004400) Train Loss: 3.7936, Train Steps/Sec: 1.10 +[2025-03-24 10:03:39] (step=0004425) Train Loss: 4.1203, Train Steps/Sec: 1.05 +[2025-03-24 10:04:01] (step=0004450) Train Loss: 3.9797, Train Steps/Sec: 1.10 +[2025-03-24 10:04:24] (step=0004475) Train Loss: 4.0027, Train Steps/Sec: 1.10 +[2025-03-24 10:04:47] (step=0004500) Train Loss: 3.9635, Train Steps/Sec: 1.10 +[2025-03-24 10:05:11] (step=0004525) Train Loss: 4.0646, Train Steps/Sec: 1.06 +[2025-03-24 10:05:33] (step=0004550) Train Loss: 3.9452, Train Steps/Sec: 1.10 +[2025-03-24 10:05:57] (step=0004575) Train Loss: 4.0051, Train Steps/Sec: 1.05 +[2025-03-24 10:06:20] (step=0004600) Train Loss: 4.0788, Train Steps/Sec: 1.10 +[2025-03-24 10:06:43] (step=0004625) Train Loss: 3.9570, Train Steps/Sec: 1.10 +[2025-03-24 10:07:05] (step=0004650) Train Loss: 3.9705, Train Steps/Sec: 1.10 +[2025-03-24 10:07:29] (step=0004675) Train Loss: 4.0453, Train Steps/Sec: 1.05 +[2025-03-24 10:07:52] (step=0004700) Train Loss: 4.0697, Train Steps/Sec: 1.10 +[2025-03-24 10:08:15] (step=0004725) Train Loss: 4.0329, Train Steps/Sec: 1.10 +[2025-03-24 10:08:38] (step=0004750) Train Loss: 3.8824, Train Steps/Sec: 1.10 +[2025-03-24 10:09:01] (step=0004775) Train Loss: 3.9376, Train Steps/Sec: 1.06 +[2025-03-24 10:09:24] (step=0004800) Train Loss: 3.9607, Train Steps/Sec: 1.10 +[2025-03-24 10:09:48] (step=0004825) Train Loss: 3.9331, Train Steps/Sec: 1.03 +[2025-03-24 10:10:11] (step=0004850) Train Loss: 4.0184, Train Steps/Sec: 1.10 +[2025-03-24 10:10:35] (step=0004875) Train Loss: 4.0057, Train Steps/Sec: 1.04 +[2025-03-24 10:10:58] (step=0004900) Train Loss: 4.0005, Train Steps/Sec: 1.10 +[2025-03-24 10:11:21] (step=0004925) Train Loss: 3.8655, Train Steps/Sec: 1.10 +[2025-03-24 10:12:11] (step=0004950) Train Loss: 4.0869, Train Steps/Sec: 0.50 +[2025-03-24 10:12:34] (step=0004975) Train Loss: 3.9027, Train Steps/Sec: 1.10 +[2025-03-24 10:12:58] (step=0005000) Train Loss: 3.9956, Train Steps/Sec: 1.05 +[2025-03-24 10:13:21] (step=0005025) Train Loss: 3.9875, Train Steps/Sec: 1.09 +[2025-03-24 10:13:43] (step=0005050) Train Loss: 3.9579, Train Steps/Sec: 1.09 +[2025-03-24 10:14:06] (step=0005075) Train Loss: 3.8947, Train Steps/Sec: 1.10 +[2025-03-24 10:14:29] (step=0005100) Train Loss: 3.9652, Train Steps/Sec: 1.10 +[2025-03-24 10:14:52] (step=0005125) Train Loss: 4.0423, Train Steps/Sec: 1.10 +[2025-03-24 10:15:15] (step=0005150) Train Loss: 4.0538, Train Steps/Sec: 1.09 +[2025-03-24 10:15:37] (step=0005175) Train Loss: 4.0805, Train Steps/Sec: 1.10 +[2025-03-24 10:16:01] (step=0005200) Train Loss: 4.0380, Train Steps/Sec: 1.05 +[2025-03-24 10:16:26] (step=0005225) Train Loss: 3.9603, Train Steps/Sec: 1.02 +[2025-03-24 10:16:49] (step=0005250) Train Loss: 3.9766, Train Steps/Sec: 1.05 +[2025-03-24 10:17:12] (step=0005275) Train Loss: 3.9327, Train Steps/Sec: 1.09 +[2025-03-24 10:17:35] (step=0005300) Train Loss: 4.0204, Train Steps/Sec: 1.10 +[2025-03-24 10:17:59] (step=0005325) Train Loss: 4.0465, Train Steps/Sec: 1.06 +[2025-03-24 10:18:21] (step=0005350) Train Loss: 3.9612, Train Steps/Sec: 1.10 +[2025-03-24 10:18:44] (step=0005375) Train Loss: 4.0689, Train Steps/Sec: 1.09 +[2025-03-24 10:19:07] (step=0005400) Train Loss: 3.9945, Train Steps/Sec: 1.09 +[2025-03-24 10:19:30] (step=0005425) Train Loss: 3.9695, Train Steps/Sec: 1.10 +[2025-03-24 10:19:53] (step=0005450) Train Loss: 3.9227, Train Steps/Sec: 1.09 +[2025-03-24 10:20:16] (step=0005475) Train Loss: 3.9511, Train Steps/Sec: 1.10 +[2025-03-24 10:20:38] (step=0005500) Train Loss: 4.0827, Train Steps/Sec: 1.10 +[2025-03-24 10:21:02] (step=0005525) Train Loss: 4.0123, Train Steps/Sec: 1.04 +[2025-03-24 10:21:25] (step=0005550) Train Loss: 4.1017, Train Steps/Sec: 1.10 +[2025-03-24 10:21:48] (step=0005575) Train Loss: 4.0200, Train Steps/Sec: 1.10 +[2025-03-24 10:22:11] (step=0005600) Train Loss: 4.0718, Train Steps/Sec: 1.10 +[2025-03-24 10:22:34] (step=0005625) Train Loss: 4.0240, Train Steps/Sec: 1.10 +[2025-03-24 10:22:56] (step=0005650) Train Loss: 4.0031, Train Steps/Sec: 1.10 +[2025-03-24 10:23:19] (step=0005675) Train Loss: 3.9967, Train Steps/Sec: 1.10 +[2025-03-24 10:23:43] (step=0005700) Train Loss: 4.0584, Train Steps/Sec: 1.05 +[2025-03-24 10:24:07] (step=0005725) Train Loss: 4.0103, Train Steps/Sec: 1.05 +[2025-03-24 10:24:30] (step=0005750) Train Loss: 3.9630, Train Steps/Sec: 1.10 +[2025-03-24 10:24:52] (step=0005775) Train Loss: 4.1181, Train Steps/Sec: 1.09 +[2025-03-24 10:25:15] (step=0005800) Train Loss: 3.9820, Train Steps/Sec: 1.09 +[2025-03-24 10:25:40] (step=0005825) Train Loss: 4.0821, Train Steps/Sec: 1.01 +[2025-03-24 10:26:03] (step=0005850) Train Loss: 3.9464, Train Steps/Sec: 1.10 +[2025-03-24 10:26:26] (step=0005875) Train Loss: 4.1394, Train Steps/Sec: 1.10 +[2025-03-24 10:26:48] (step=0005900) Train Loss: 3.9993, Train Steps/Sec: 1.10 +[2025-03-24 10:27:12] (step=0005925) Train Loss: 4.0358, Train Steps/Sec: 1.06 +[2025-03-24 10:27:36] (step=0005950) Train Loss: 3.9850, Train Steps/Sec: 1.05 +[2025-03-24 10:27:59] (step=0005975) Train Loss: 4.1222, Train Steps/Sec: 1.10 +[2025-03-24 10:28:21] (step=0006000) Train Loss: 4.1236, Train Steps/Sec: 1.10 +[2025-03-24 10:28:21] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 10:37:18] Finish Eval in 6000 steps... +[2025-03-24 10:37:42] (step=0006025) Train Loss: 4.1197, Train Steps/Sec: 0.04 +[2025-03-24 10:38:04] (step=0006050) Train Loss: 4.1199, Train Steps/Sec: 1.10 +[2025-03-24 10:38:27] (step=0006075) Train Loss: 3.8991, Train Steps/Sec: 1.10 +[2025-03-24 10:38:51] (step=0006100) Train Loss: 3.8539, Train Steps/Sec: 1.05 +[2025-03-24 10:39:14] (step=0006125) Train Loss: 4.1425, Train Steps/Sec: 1.10 +[2025-03-24 10:39:36] (step=0006150) Train Loss: 4.0484, Train Steps/Sec: 1.10 +[2025-03-24 10:39:59] (step=0006175) Train Loss: 4.1352, Train Steps/Sec: 1.10 +[2025-03-24 10:40:22] (step=0006200) Train Loss: 3.9956, Train Steps/Sec: 1.10 +[2025-03-24 10:40:46] (step=0006225) Train Loss: 4.1135, Train Steps/Sec: 1.06 +[2025-03-24 10:41:10] (step=0006250) Train Loss: 4.0769, Train Steps/Sec: 1.04 +[2025-03-24 10:41:32] (step=0006275) Train Loss: 3.9527, Train Steps/Sec: 1.10 +[2025-03-24 10:41:55] (step=0006300) Train Loss: 4.0232, Train Steps/Sec: 1.10 +[2025-03-24 10:42:18] (step=0006325) Train Loss: 3.9679, Train Steps/Sec: 1.10 +[2025-03-24 10:42:41] (step=0006350) Train Loss: 3.9505, Train Steps/Sec: 1.10 +[2025-03-24 10:43:04] (step=0006375) Train Loss: 4.0787, Train Steps/Sec: 1.06 +[2025-03-24 10:43:27] (step=0006400) Train Loss: 4.0765, Train Steps/Sec: 1.09 +[2025-03-24 10:43:51] (step=0006425) Train Loss: 3.9959, Train Steps/Sec: 1.06 +[2025-03-24 10:44:13] (step=0006450) Train Loss: 4.0360, Train Steps/Sec: 1.10 +[2025-03-24 10:44:36] (step=0006475) Train Loss: 3.9170, Train Steps/Sec: 1.09 +[2025-03-24 10:44:59] (step=0006500) Train Loss: 4.0314, Train Steps/Sec: 1.09 +[2025-03-24 10:45:23] (step=0006525) Train Loss: 4.0858, Train Steps/Sec: 1.05 +[2025-03-24 10:45:46] (step=0006550) Train Loss: 3.9782, Train Steps/Sec: 1.10 +[2025-03-24 10:46:10] (step=0006575) Train Loss: 4.0028, Train Steps/Sec: 1.05 +[2025-03-24 10:46:34] (step=0006600) Train Loss: 4.0054, Train Steps/Sec: 1.04 +[2025-03-24 10:46:57] (step=0006625) Train Loss: 4.0480, Train Steps/Sec: 1.10 +[2025-03-24 10:47:19] (step=0006650) Train Loss: 4.0904, Train Steps/Sec: 1.10 +[2025-03-24 10:47:42] (step=0006675) Train Loss: 4.1151, Train Steps/Sec: 1.10 +[2025-03-24 10:48:07] (step=0006700) Train Loss: 3.9616, Train Steps/Sec: 1.00 +[2025-03-24 10:48:30] (step=0006725) Train Loss: 4.1074, Train Steps/Sec: 1.10 +[2025-03-24 10:48:53] (step=0006750) Train Loss: 3.9773, Train Steps/Sec: 1.09 +[2025-03-24 10:49:15] (step=0006775) Train Loss: 3.9558, Train Steps/Sec: 1.10 +[2025-03-24 10:49:38] (step=0006800) Train Loss: 4.1237, Train Steps/Sec: 1.10 +[2025-03-24 10:50:01] (step=0006825) Train Loss: 4.0116, Train Steps/Sec: 1.10 +[2025-03-24 10:50:24] (step=0006850) Train Loss: 4.1066, Train Steps/Sec: 1.10 +[2025-03-24 10:50:47] (step=0006875) Train Loss: 3.9729, Train Steps/Sec: 1.10 +[2025-03-24 10:51:10] (step=0006900) Train Loss: 4.0953, Train Steps/Sec: 1.09 +[2025-03-24 10:51:33] (step=0006925) Train Loss: 4.0604, Train Steps/Sec: 1.06 +[2025-03-24 10:51:56] (step=0006950) Train Loss: 4.0401, Train Steps/Sec: 1.10 +[2025-03-24 10:52:19] (step=0006975) Train Loss: 4.0635, Train Steps/Sec: 1.06 +[2025-03-24 10:52:42] (step=0007000) Train Loss: 4.1614, Train Steps/Sec: 1.10 +[2025-03-24 10:53:05] (step=0007025) Train Loss: 4.0793, Train Steps/Sec: 1.10 +[2025-03-24 10:53:29] (step=0007050) Train Loss: 4.0456, Train Steps/Sec: 1.05 +[2025-03-24 10:53:52] (step=0007075) Train Loss: 4.0206, Train Steps/Sec: 1.06 +[2025-03-24 10:54:15] (step=0007100) Train Loss: 4.0632, Train Steps/Sec: 1.10 +[2025-03-24 10:54:38] (step=0007125) Train Loss: 4.0561, Train Steps/Sec: 1.10 +[2025-03-24 10:55:01] (step=0007150) Train Loss: 4.0710, Train Steps/Sec: 1.10 +[2025-03-24 10:55:24] (step=0007175) Train Loss: 4.0311, Train Steps/Sec: 1.10 +[2025-03-24 10:55:46] (step=0007200) Train Loss: 3.9657, Train Steps/Sec: 1.10 +[2025-03-24 10:56:09] (step=0007225) Train Loss: 4.1051, Train Steps/Sec: 1.10 +[2025-03-24 10:56:32] (step=0007250) Train Loss: 4.0677, Train Steps/Sec: 1.10 +[2025-03-24 10:56:55] (step=0007275) Train Loss: 3.9572, Train Steps/Sec: 1.10 +[2025-03-24 10:57:17] (step=0007300) Train Loss: 3.9911, Train Steps/Sec: 1.10 +[2025-03-24 10:57:40] (step=0007325) Train Loss: 4.0189, Train Steps/Sec: 1.10 +[2025-03-24 10:58:03] (step=0007350) Train Loss: 4.1188, Train Steps/Sec: 1.10 +[2025-03-24 10:58:26] (step=0007375) Train Loss: 4.0648, Train Steps/Sec: 1.10 +[2025-03-24 10:58:48] (step=0007400) Train Loss: 3.9744, Train Steps/Sec: 1.10 +[2025-03-24 10:59:14] (step=0007425) Train Loss: 4.0896, Train Steps/Sec: 0.97 +[2025-03-24 10:59:38] (step=0007450) Train Loss: 4.1629, Train Steps/Sec: 1.05 +[2025-03-24 11:00:01] (step=0007475) Train Loss: 3.9536, Train Steps/Sec: 1.10 +[2025-03-24 11:00:24] (step=0007500) Train Loss: 4.0095, Train Steps/Sec: 1.10 +[2025-03-24 11:00:46] (step=0007525) Train Loss: 4.0199, Train Steps/Sec: 1.10 +[2025-03-24 11:01:09] (step=0007550) Train Loss: 3.9727, Train Steps/Sec: 1.09 +[2025-03-24 11:01:32] (step=0007575) Train Loss: 4.0295, Train Steps/Sec: 1.10 +[2025-03-24 11:01:56] (step=0007600) Train Loss: 4.0580, Train Steps/Sec: 1.06 +[2025-03-24 11:02:19] (step=0007625) Train Loss: 3.9957, Train Steps/Sec: 1.05 +[2025-03-24 11:03:09] (step=0007650) Train Loss: 4.1605, Train Steps/Sec: 0.51 +[2025-03-24 11:03:31] (step=0007675) Train Loss: 4.0625, Train Steps/Sec: 1.10 +[2025-03-24 11:03:54] (step=0007700) Train Loss: 4.0028, Train Steps/Sec: 1.09 +[2025-03-24 11:04:17] (step=0007725) Train Loss: 4.0138, Train Steps/Sec: 1.10 +[2025-03-24 11:04:41] (step=0007750) Train Loss: 4.1545, Train Steps/Sec: 1.06 +[2025-03-24 11:05:03] (step=0007775) Train Loss: 4.0544, Train Steps/Sec: 1.10 +[2025-03-24 11:05:27] (step=0007800) Train Loss: 4.0681, Train Steps/Sec: 1.05 +[2025-03-24 11:05:50] (step=0007825) Train Loss: 3.9908, Train Steps/Sec: 1.10 +[2025-03-24 11:06:13] (step=0007850) Train Loss: 4.1568, Train Steps/Sec: 1.10 +[2025-03-24 11:06:36] (step=0007875) Train Loss: 4.0122, Train Steps/Sec: 1.10 +[2025-03-24 11:06:58] (step=0007900) Train Loss: 4.1159, Train Steps/Sec: 1.10 +[2025-03-24 11:07:22] (step=0007925) Train Loss: 4.1340, Train Steps/Sec: 1.04 +[2025-03-24 11:08:12] (step=0007950) Train Loss: 4.0395, Train Steps/Sec: 0.50 +[2025-03-24 11:08:35] (step=0007975) Train Loss: 3.9756, Train Steps/Sec: 1.10 +[2025-03-24 11:08:58] (step=0008000) Train Loss: 4.0118, Train Steps/Sec: 1.10 +[2025-03-24 11:08:58] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 11:17:59] Finish Eval in 8000 steps... +[2025-03-24 11:18:20] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0008000.pt +[2025-03-24 11:18:22] Removed old checkpoint: checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0004000.pt +[2025-03-24 11:18:45] (step=0008025) Train Loss: 4.0416, Train Steps/Sec: 0.04 +[2025-03-24 11:19:09] (step=0008050) Train Loss: 4.1494, Train Steps/Sec: 1.05 +[2025-03-24 11:19:32] (step=0008075) Train Loss: 4.0250, Train Steps/Sec: 1.10 +[2025-03-24 11:19:55] (step=0008100) Train Loss: 3.9922, Train Steps/Sec: 1.07 +[2025-03-24 11:20:18] (step=0008125) Train Loss: 3.9529, Train Steps/Sec: 1.10 +[2025-03-24 11:20:40] (step=0008150) Train Loss: 4.0346, Train Steps/Sec: 1.10 +[2025-03-24 11:21:03] (step=0008175) Train Loss: 4.0278, Train Steps/Sec: 1.10 +[2025-03-24 11:21:26] (step=0008200) Train Loss: 4.0293, Train Steps/Sec: 1.10 +[2025-03-24 11:21:49] (step=0008225) Train Loss: 4.0651, Train Steps/Sec: 1.10 +[2025-03-24 11:22:11] (step=0008250) Train Loss: 4.1205, Train Steps/Sec: 1.10 +[2025-03-24 11:22:34] (step=0008275) Train Loss: 3.9214, Train Steps/Sec: 1.10 +[2025-03-24 11:22:57] (step=0008300) Train Loss: 4.2208, Train Steps/Sec: 1.10 +[2025-03-24 11:23:20] (step=0008325) Train Loss: 4.1320, Train Steps/Sec: 1.07 +[2025-03-24 11:24:10] (step=0008350) Train Loss: 3.9979, Train Steps/Sec: 0.51 +[2025-03-24 11:24:33] (step=0008375) Train Loss: 4.0285, Train Steps/Sec: 1.05 +[2025-03-24 11:24:56] (step=0008400) Train Loss: 4.1201, Train Steps/Sec: 1.10 +[2025-03-24 11:25:19] (step=0008425) Train Loss: 4.0249, Train Steps/Sec: 1.10 +[2025-03-24 11:25:42] (step=0008450) Train Loss: 3.9572, Train Steps/Sec: 1.10 +[2025-03-24 11:26:06] (step=0008475) Train Loss: 4.0333, Train Steps/Sec: 1.05 +[2025-03-24 11:26:28] (step=0008500) Train Loss: 4.0053, Train Steps/Sec: 1.10 +[2025-03-24 11:26:51] (step=0008525) Train Loss: 4.0563, Train Steps/Sec: 1.10 +[2025-03-24 11:27:14] (step=0008550) Train Loss: 4.0645, Train Steps/Sec: 1.10 +[2025-03-24 11:27:37] (step=0008575) Train Loss: 4.0916, Train Steps/Sec: 1.10 +[2025-03-24 11:28:00] (step=0008600) Train Loss: 4.1081, Train Steps/Sec: 1.05 +[2025-03-24 11:28:23] (step=0008625) Train Loss: 3.9873, Train Steps/Sec: 1.10 +[2025-03-24 11:28:46] (step=0008650) Train Loss: 4.0245, Train Steps/Sec: 1.10 +[2025-03-24 11:29:09] (step=0008675) Train Loss: 3.9900, Train Steps/Sec: 1.10 +[2025-03-24 11:29:32] (step=0008700) Train Loss: 4.1167, Train Steps/Sec: 1.06 +[2025-03-24 11:29:55] (step=0008725) Train Loss: 4.1297, Train Steps/Sec: 1.10 +[2025-03-24 11:30:18] (step=0008750) Train Loss: 3.9935, Train Steps/Sec: 1.10 +[2025-03-24 11:30:41] (step=0008775) Train Loss: 4.0182, Train Steps/Sec: 1.10 +[2025-03-24 11:31:03] (step=0008800) Train Loss: 4.0240, Train Steps/Sec: 1.10 +[2025-03-24 11:31:26] (step=0008825) Train Loss: 4.0936, Train Steps/Sec: 1.10 +[2025-03-24 11:31:50] (step=0008850) Train Loss: 3.9998, Train Steps/Sec: 1.06 +[2025-03-24 11:32:12] (step=0008875) Train Loss: 4.0609, Train Steps/Sec: 1.10 +[2025-03-24 11:32:36] (step=0008900) Train Loss: 3.9739, Train Steps/Sec: 1.06 +[2025-03-24 11:33:00] (step=0008925) Train Loss: 3.9669, Train Steps/Sec: 1.04 +[2025-03-24 11:33:23] (step=0008950) Train Loss: 3.9677, Train Steps/Sec: 1.10 +[2025-03-24 11:33:46] (step=0008975) Train Loss: 3.9741, Train Steps/Sec: 1.10 +[2025-03-24 11:34:08] (step=0009000) Train Loss: 4.0596, Train Steps/Sec: 1.10 +[2025-03-24 11:34:31] (step=0009025) Train Loss: 4.0945, Train Steps/Sec: 1.10 +[2025-03-24 11:34:55] (step=0009050) Train Loss: 4.0836, Train Steps/Sec: 1.05 +[2025-03-24 11:35:19] (step=0009075) Train Loss: 4.0025, Train Steps/Sec: 1.05 +[2025-03-24 11:35:41] (step=0009100) Train Loss: 4.0727, Train Steps/Sec: 1.10 +[2025-03-24 11:36:04] (step=0009125) Train Loss: 4.0347, Train Steps/Sec: 1.10 +[2025-03-24 11:36:28] (step=0009150) Train Loss: 4.0263, Train Steps/Sec: 1.05 +[2025-03-24 11:36:51] (step=0009175) Train Loss: 4.0724, Train Steps/Sec: 1.10 +[2025-03-24 11:37:15] (step=0009200) Train Loss: 4.0422, Train Steps/Sec: 1.05 +[2025-03-24 11:37:37] (step=0009225) Train Loss: 4.0209, Train Steps/Sec: 1.10 +[2025-03-24 11:38:00] (step=0009250) Train Loss: 3.9129, Train Steps/Sec: 1.10 +[2025-03-24 11:38:23] (step=0009275) Train Loss: 4.0672, Train Steps/Sec: 1.10 +[2025-03-24 11:38:46] (step=0009300) Train Loss: 3.9437, Train Steps/Sec: 1.10 +[2025-03-24 11:39:09] (step=0009325) Train Loss: 3.9146, Train Steps/Sec: 1.06 +[2025-03-24 11:39:32] (step=0009350) Train Loss: 4.1373, Train Steps/Sec: 1.10 +[2025-03-24 11:39:55] (step=0009375) Train Loss: 3.9998, Train Steps/Sec: 1.10 +[2025-03-24 11:40:17] (step=0009400) Train Loss: 3.9705, Train Steps/Sec: 1.10 +[2025-03-24 11:40:40] (step=0009425) Train Loss: 3.8901, Train Steps/Sec: 1.09 +[2025-03-24 11:41:03] (step=0009450) Train Loss: 4.0317, Train Steps/Sec: 1.10 +[2025-03-24 11:41:27] (step=0009475) Train Loss: 3.9438, Train Steps/Sec: 1.06 +[2025-03-24 11:41:50] (step=0009500) Train Loss: 4.1032, Train Steps/Sec: 1.10 +[2025-03-24 11:42:12] (step=0009525) Train Loss: 4.0113, Train Steps/Sec: 1.10 +[2025-03-24 11:42:35] (step=0009550) Train Loss: 3.9810, Train Steps/Sec: 1.10 +[2025-03-24 11:43:00] (step=0009575) Train Loss: 3.9932, Train Steps/Sec: 1.01 +[2025-03-24 11:43:23] (step=0009600) Train Loss: 4.0821, Train Steps/Sec: 1.10 +[2025-03-24 11:43:46] (step=0009625) Train Loss: 4.1046, Train Steps/Sec: 1.05 +[2025-03-24 11:44:09] (step=0009650) Train Loss: 4.0197, Train Steps/Sec: 1.10 +[2025-03-24 11:44:33] (step=0009675) Train Loss: 4.1050, Train Steps/Sec: 1.05 +[2025-03-24 11:44:56] (step=0009700) Train Loss: 3.9146, Train Steps/Sec: 1.10 +[2025-03-24 11:45:18] (step=0009725) Train Loss: 3.9339, Train Steps/Sec: 1.10 +[2025-03-24 11:45:41] (step=0009750) Train Loss: 3.9330, Train Steps/Sec: 1.10 +[2025-03-24 11:46:04] (step=0009775) Train Loss: 4.1189, Train Steps/Sec: 1.10 +[2025-03-24 11:46:29] (step=0009800) Train Loss: 4.0059, Train Steps/Sec: 1.02 +[2025-03-24 11:46:51] (step=0009825) Train Loss: 3.9947, Train Steps/Sec: 1.10 +[2025-03-24 11:47:15] (step=0009850) Train Loss: 4.1284, Train Steps/Sec: 1.05 +[2025-03-24 11:47:38] (step=0009875) Train Loss: 4.1022, Train Steps/Sec: 1.10 +[2025-03-24 11:48:01] (step=0009900) Train Loss: 4.0145, Train Steps/Sec: 1.10 +[2025-03-24 11:48:24] (step=0009925) Train Loss: 4.1120, Train Steps/Sec: 1.05 +[2025-03-24 11:48:47] (step=0009950) Train Loss: 3.9351, Train Steps/Sec: 1.10 +[2025-03-24 11:49:10] (step=0009975) Train Loss: 3.9568, Train Steps/Sec: 1.10 +[2025-03-24 11:49:33] (step=0010000) Train Loss: 3.9528, Train Steps/Sec: 1.10 +[2025-03-24 11:49:33] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 11:58:36] Finish Eval in 10000 steps... +[2025-03-24 11:58:59] (step=0010025) Train Loss: 3.9307, Train Steps/Sec: 0.04 +[2025-03-24 11:59:22] (step=0010050) Train Loss: 3.8784, Train Steps/Sec: 1.09 +[2025-03-24 11:59:45] (step=0010075) Train Loss: 4.0125, Train Steps/Sec: 1.10 +[2025-03-24 12:00:07] (step=0010100) Train Loss: 4.0688, Train Steps/Sec: 1.10 +[2025-03-24 12:00:31] (step=0010125) Train Loss: 4.0692, Train Steps/Sec: 1.06 +[2025-03-24 12:00:55] (step=0010150) Train Loss: 4.0866, Train Steps/Sec: 1.05 +[2025-03-24 12:01:18] (step=0010175) Train Loss: 3.9183, Train Steps/Sec: 1.10 +[2025-03-24 12:01:41] (step=0010200) Train Loss: 3.9231, Train Steps/Sec: 1.05 +[2025-03-24 12:02:04] (step=0010225) Train Loss: 4.0445, Train Steps/Sec: 1.10 +[2025-03-24 12:02:27] (step=0010250) Train Loss: 4.0809, Train Steps/Sec: 1.10 +[2025-03-24 12:02:50] (step=0010275) Train Loss: 4.0954, Train Steps/Sec: 1.10 +[2025-03-24 12:03:13] (step=0010300) Train Loss: 3.9544, Train Steps/Sec: 1.10 +[2025-03-24 12:03:35] (step=0010325) Train Loss: 4.0629, Train Steps/Sec: 1.10 +[2025-03-24 12:03:58] (step=0010350) Train Loss: 3.9266, Train Steps/Sec: 1.10 +[2025-03-24 12:04:21] (step=0010375) Train Loss: 3.9386, Train Steps/Sec: 1.10 +[2025-03-24 12:04:44] (step=0010400) Train Loss: 3.9641, Train Steps/Sec: 1.10 +[2025-03-24 12:05:06] (step=0010425) Train Loss: 3.9107, Train Steps/Sec: 1.10 +[2025-03-24 12:05:30] (step=0010450) Train Loss: 3.9594, Train Steps/Sec: 1.10 +[2025-03-24 12:05:54] (step=0010475) Train Loss: 4.0157, Train Steps/Sec: 1.07 +[2025-03-24 12:06:17] (step=0010500) Train Loss: 4.0387, Train Steps/Sec: 1.10 +[2025-03-24 12:06:39] (step=0010525) Train Loss: 4.0182, Train Steps/Sec: 1.10 +[2025-03-24 12:07:03] (step=0010550) Train Loss: 4.1250, Train Steps/Sec: 1.05 +[2025-03-24 12:07:26] (step=0010575) Train Loss: 3.9902, Train Steps/Sec: 1.10 +[2025-03-24 12:07:49] (step=0010600) Train Loss: 3.8836, Train Steps/Sec: 1.10 +[2025-03-24 12:08:11] (step=0010625) Train Loss: 3.9598, Train Steps/Sec: 1.10 +[2025-03-24 12:08:34] (step=0010650) Train Loss: 3.9453, Train Steps/Sec: 1.10 +[2025-03-24 12:08:59] (step=0010675) Train Loss: 3.9836, Train Steps/Sec: 1.00 +[2025-03-24 12:09:22] (step=0010700) Train Loss: 3.8844, Train Steps/Sec: 1.10 +[2025-03-24 12:09:45] (step=0010725) Train Loss: 4.0067, Train Steps/Sec: 1.06 +[2025-03-24 12:10:08] (step=0010750) Train Loss: 3.9032, Train Steps/Sec: 1.10 +[2025-03-24 12:10:31] (step=0010775) Train Loss: 3.9612, Train Steps/Sec: 1.10 +[2025-03-24 12:10:54] (step=0010800) Train Loss: 3.8929, Train Steps/Sec: 1.10 +[2025-03-24 12:11:17] (step=0010825) Train Loss: 3.9190, Train Steps/Sec: 1.10 +[2025-03-24 12:11:39] (step=0010850) Train Loss: 4.1196, Train Steps/Sec: 1.10 +[2025-03-24 12:12:02] (step=0010875) Train Loss: 3.8741, Train Steps/Sec: 1.10 +[2025-03-24 12:12:25] (step=0010900) Train Loss: 3.9729, Train Steps/Sec: 1.10 +[2025-03-24 12:12:48] (step=0010925) Train Loss: 4.0194, Train Steps/Sec: 1.10 +[2025-03-24 12:13:10] (step=0010950) Train Loss: 3.8576, Train Steps/Sec: 1.10 +[2025-03-24 12:13:34] (step=0010975) Train Loss: 3.9707, Train Steps/Sec: 1.05 +[2025-03-24 12:13:58] (step=0011000) Train Loss: 4.0065, Train Steps/Sec: 1.05 +[2025-03-24 12:14:21] (step=0011025) Train Loss: 4.0142, Train Steps/Sec: 1.10 +[2025-03-24 12:14:44] (step=0011050) Train Loss: 3.9616, Train Steps/Sec: 1.10 +[2025-03-24 12:15:06] (step=0011075) Train Loss: 3.8998, Train Steps/Sec: 1.10 +[2025-03-24 12:15:30] (step=0011100) Train Loss: 3.9133, Train Steps/Sec: 1.05 +[2025-03-24 12:15:53] (step=0011125) Train Loss: 3.9706, Train Steps/Sec: 1.10 +[2025-03-24 12:16:16] (step=0011150) Train Loss: 4.0264, Train Steps/Sec: 1.07 +[2025-03-24 12:16:39] (step=0011175) Train Loss: 3.8168, Train Steps/Sec: 1.10 +[2025-03-24 12:17:02] (step=0011200) Train Loss: 4.0246, Train Steps/Sec: 1.10 +[2025-03-24 12:17:26] (step=0011225) Train Loss: 3.9986, Train Steps/Sec: 1.05 +[2025-03-24 12:17:49] (step=0011250) Train Loss: 4.0381, Train Steps/Sec: 1.06 +[2025-03-24 12:18:12] (step=0011275) Train Loss: 3.9025, Train Steps/Sec: 1.10 +[2025-03-24 12:18:35] (step=0011300) Train Loss: 3.8730, Train Steps/Sec: 1.10 +[2025-03-24 12:18:58] (step=0011325) Train Loss: 4.0631, Train Steps/Sec: 1.10 +[2025-03-24 12:19:21] (step=0011350) Train Loss: 3.9104, Train Steps/Sec: 1.06 +[2025-03-24 12:19:44] (step=0011375) Train Loss: 3.9969, Train Steps/Sec: 1.10 +[2025-03-24 12:20:07] (step=0011400) Train Loss: 3.8362, Train Steps/Sec: 1.10 +[2025-03-24 12:20:29] (step=0011425) Train Loss: 3.9240, Train Steps/Sec: 1.10 +[2025-03-24 12:20:52] (step=0011450) Train Loss: 3.9425, Train Steps/Sec: 1.10 +[2025-03-24 12:21:16] (step=0011475) Train Loss: 3.9817, Train Steps/Sec: 1.05 +[2025-03-24 12:21:39] (step=0011500) Train Loss: 4.0620, Train Steps/Sec: 1.10 +[2025-03-24 12:22:01] (step=0011525) Train Loss: 3.9479, Train Steps/Sec: 1.10 +[2025-03-24 12:22:24] (step=0011550) Train Loss: 3.9652, Train Steps/Sec: 1.10 +[2025-03-24 12:22:47] (step=0011575) Train Loss: 3.9575, Train Steps/Sec: 1.10 +[2025-03-24 12:23:11] (step=0011600) Train Loss: 3.9184, Train Steps/Sec: 1.06 +[2025-03-24 12:23:33] (step=0011625) Train Loss: 3.9976, Train Steps/Sec: 1.09 +[2025-03-24 12:23:56] (step=0011650) Train Loss: 4.0163, Train Steps/Sec: 1.10 +[2025-03-24 12:24:19] (step=0011675) Train Loss: 3.9339, Train Steps/Sec: 1.10 +[2025-03-24 12:24:43] (step=0011700) Train Loss: 4.0896, Train Steps/Sec: 1.05 +[2025-03-24 12:25:07] (step=0011725) Train Loss: 3.9238, Train Steps/Sec: 1.05 +[2025-03-24 12:25:30] (step=0011750) Train Loss: 3.8668, Train Steps/Sec: 1.05 +[2025-03-24 12:25:53] (step=0011775) Train Loss: 3.9434, Train Steps/Sec: 1.10 +[2025-03-24 12:26:17] (step=0011800) Train Loss: 3.9786, Train Steps/Sec: 1.05 +[2025-03-24 12:26:41] (step=0011825) Train Loss: 3.9668, Train Steps/Sec: 1.05 +[2025-03-24 12:27:04] (step=0011850) Train Loss: 3.9481, Train Steps/Sec: 1.10 +[2025-03-24 12:27:26] (step=0011875) Train Loss: 3.9912, Train Steps/Sec: 1.10 +[2025-03-24 12:27:49] (step=0011900) Train Loss: 3.8915, Train Steps/Sec: 1.10 +[2025-03-24 12:28:12] (step=0011925) Train Loss: 3.9910, Train Steps/Sec: 1.10 +[2025-03-24 12:28:35] (step=0011950) Train Loss: 3.9491, Train Steps/Sec: 1.09 +[2025-03-24 12:28:58] (step=0011975) Train Loss: 3.8943, Train Steps/Sec: 1.06 +[2025-03-24 12:29:21] (step=0012000) Train Loss: 3.9327, Train Steps/Sec: 1.10 +[2025-03-24 12:29:21] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 12:38:23] Finish Eval in 12000 steps... +[2025-03-24 12:38:41] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0012000.pt +[2025-03-24 12:38:43] Removed old checkpoint: checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0008000.pt +[2025-03-24 12:39:06] (step=0012025) Train Loss: 3.9945, Train Steps/Sec: 0.04 +[2025-03-24 12:39:29] (step=0012050) Train Loss: 3.9142, Train Steps/Sec: 1.10 +[2025-03-24 12:39:52] (step=0012075) Train Loss: 4.0554, Train Steps/Sec: 1.10 +[2025-03-24 12:40:14] (step=0012100) Train Loss: 3.9243, Train Steps/Sec: 1.10 +[2025-03-24 12:40:37] (step=0012125) Train Loss: 3.9639, Train Steps/Sec: 1.10 +[2025-03-24 12:41:00] (step=0012150) Train Loss: 3.8675, Train Steps/Sec: 1.10 +[2025-03-24 12:41:23] (step=0012175) Train Loss: 3.8129, Train Steps/Sec: 1.10 +[2025-03-24 12:41:46] (step=0012200) Train Loss: 3.8501, Train Steps/Sec: 1.10 +[2025-03-24 12:42:08] (step=0012225) Train Loss: 3.9917, Train Steps/Sec: 1.10 +[2025-03-24 12:42:31] (step=0012250) Train Loss: 3.8778, Train Steps/Sec: 1.10 +[2025-03-24 12:42:55] (step=0012275) Train Loss: 3.9570, Train Steps/Sec: 1.05 +[2025-03-24 12:43:18] (step=0012300) Train Loss: 3.9565, Train Steps/Sec: 1.06 +[2025-03-24 12:43:41] (step=0012325) Train Loss: 3.8473, Train Steps/Sec: 1.10 +[2025-03-24 12:44:04] (step=0012350) Train Loss: 4.0417, Train Steps/Sec: 1.10 +[2025-03-24 12:44:27] (step=0012375) Train Loss: 4.0909, Train Steps/Sec: 1.10 +[2025-03-24 12:44:51] (step=0012400) Train Loss: 3.9523, Train Steps/Sec: 1.01 +[2025-03-24 12:45:14] (step=0012425) Train Loss: 3.9533, Train Steps/Sec: 1.10 +[2025-03-24 12:45:37] (step=0012450) Train Loss: 3.9483, Train Steps/Sec: 1.10 +[2025-03-24 12:46:00] (step=0012475) Train Loss: 4.0402, Train Steps/Sec: 1.09 +[2025-03-24 12:46:23] (step=0012500) Train Loss: 4.0832, Train Steps/Sec: 1.10 +[2025-03-24 12:46:46] (step=0012525) Train Loss: 3.9001, Train Steps/Sec: 1.05 +[2025-03-24 12:47:09] (step=0012550) Train Loss: 3.8924, Train Steps/Sec: 1.10 +[2025-03-24 12:47:33] (step=0012575) Train Loss: 4.0073, Train Steps/Sec: 1.05 +[2025-03-24 12:47:56] (step=0012600) Train Loss: 3.9703, Train Steps/Sec: 1.10 +[2025-03-24 12:48:19] (step=0012625) Train Loss: 3.8759, Train Steps/Sec: 1.06 +[2025-03-24 12:48:42] (step=0012650) Train Loss: 3.9104, Train Steps/Sec: 1.10 +[2025-03-24 12:49:05] (step=0012675) Train Loss: 3.9331, Train Steps/Sec: 1.10 +[2025-03-24 12:49:29] (step=0012700) Train Loss: 3.9353, Train Steps/Sec: 1.04 +[2025-03-24 12:49:52] (step=0012725) Train Loss: 3.9271, Train Steps/Sec: 1.10 +[2025-03-24 12:50:14] (step=0012750) Train Loss: 3.9157, Train Steps/Sec: 1.10 +[2025-03-24 12:50:37] (step=0012775) Train Loss: 3.9823, Train Steps/Sec: 1.09 +[2025-03-24 12:51:00] (step=0012800) Train Loss: 3.9138, Train Steps/Sec: 1.09 +[2025-03-24 12:51:23] (step=0012825) Train Loss: 4.0440, Train Steps/Sec: 1.10 +[2025-03-24 12:51:46] (step=0012850) Train Loss: 4.0150, Train Steps/Sec: 1.10 +[2025-03-24 12:52:08] (step=0012875) Train Loss: 3.8485, Train Steps/Sec: 1.10 +[2025-03-24 12:52:32] (step=0012900) Train Loss: 4.0198, Train Steps/Sec: 1.06 +[2025-03-24 12:52:55] (step=0012925) Train Loss: 3.9467, Train Steps/Sec: 1.10 +[2025-03-24 12:53:17] (step=0012950) Train Loss: 3.9934, Train Steps/Sec: 1.10 +[2025-03-24 12:53:41] (step=0012975) Train Loss: 3.9119, Train Steps/Sec: 1.07 +[2025-03-24 12:54:05] (step=0013000) Train Loss: 3.8157, Train Steps/Sec: 1.05 +[2025-03-24 12:54:29] (step=0013025) Train Loss: 3.9825, Train Steps/Sec: 1.05 +[2025-03-24 12:54:51] (step=0013050) Train Loss: 3.9612, Train Steps/Sec: 1.10 +[2025-03-24 12:55:14] (step=0013075) Train Loss: 3.9100, Train Steps/Sec: 1.10 +[2025-03-24 12:55:37] (step=0013100) Train Loss: 3.9186, Train Steps/Sec: 1.10 +[2025-03-24 12:55:59] (step=0013125) Train Loss: 3.9599, Train Steps/Sec: 1.10 +[2025-03-24 12:56:22] (step=0013150) Train Loss: 3.8642, Train Steps/Sec: 1.10 +[2025-03-24 12:56:45] (step=0013175) Train Loss: 3.9165, Train Steps/Sec: 1.10 +[2025-03-24 12:57:08] (step=0013200) Train Loss: 3.9382, Train Steps/Sec: 1.10 +[2025-03-24 12:57:30] (step=0013225) Train Loss: 3.9720, Train Steps/Sec: 1.10 +[2025-03-24 12:57:54] (step=0013250) Train Loss: 3.8577, Train Steps/Sec: 1.05 +[2025-03-24 12:58:17] (step=0013275) Train Loss: 3.9132, Train Steps/Sec: 1.10 +[2025-03-24 12:58:40] (step=0013300) Train Loss: 3.9652, Train Steps/Sec: 1.10 +[2025-03-24 12:59:02] (step=0013325) Train Loss: 4.0022, Train Steps/Sec: 1.10 +[2025-03-24 12:59:25] (step=0013350) Train Loss: 4.0367, Train Steps/Sec: 1.10 +[2025-03-24 12:59:48] (step=0013375) Train Loss: 3.9371, Train Steps/Sec: 1.10 +[2025-03-24 13:00:13] (step=0013400) Train Loss: 3.8963, Train Steps/Sec: 1.00 +[2025-03-24 13:00:37] (step=0013425) Train Loss: 3.9196, Train Steps/Sec: 1.05 +[2025-03-24 13:01:00] (step=0013450) Train Loss: 3.9423, Train Steps/Sec: 1.07 +[2025-03-24 13:01:23] (step=0013475) Train Loss: 3.9701, Train Steps/Sec: 1.10 +[2025-03-24 13:01:45] (step=0013500) Train Loss: 3.9220, Train Steps/Sec: 1.10 +[2025-03-24 13:02:08] (step=0013525) Train Loss: 3.9357, Train Steps/Sec: 1.10 +[2025-03-24 13:02:31] (step=0013550) Train Loss: 3.8928, Train Steps/Sec: 1.10 +[2025-03-24 13:02:54] (step=0013575) Train Loss: 3.9551, Train Steps/Sec: 1.10 +[2025-03-24 13:03:17] (step=0013600) Train Loss: 3.9482, Train Steps/Sec: 1.06 +[2025-03-24 13:03:41] (step=0013625) Train Loss: 3.8540, Train Steps/Sec: 1.05 +[2025-03-24 13:04:04] (step=0013650) Train Loss: 3.9671, Train Steps/Sec: 1.10 +[2025-03-24 13:04:26] (step=0013675) Train Loss: 3.9184, Train Steps/Sec: 1.10 +[2025-03-24 13:04:50] (step=0013700) Train Loss: 3.9963, Train Steps/Sec: 1.05 +[2025-03-24 13:05:13] (step=0013725) Train Loss: 3.9830, Train Steps/Sec: 1.10 +[2025-03-24 13:05:36] (step=0013750) Train Loss: 3.9521, Train Steps/Sec: 1.10 +[2025-03-24 13:06:08] (step=0013775) Train Loss: 3.9510, Train Steps/Sec: 0.76 +[2025-03-24 13:06:31] (step=0013800) Train Loss: 3.8771, Train Steps/Sec: 1.10 +[2025-03-24 13:06:54] (step=0013825) Train Loss: 3.9367, Train Steps/Sec: 1.10 +[2025-03-24 13:07:17] (step=0013850) Train Loss: 3.8606, Train Steps/Sec: 1.10 +[2025-03-24 13:07:39] (step=0013875) Train Loss: 3.9089, Train Steps/Sec: 1.10 +[2025-03-24 13:08:02] (step=0013900) Train Loss: 3.8423, Train Steps/Sec: 1.10 +[2025-03-24 13:08:25] (step=0013925) Train Loss: 3.9317, Train Steps/Sec: 1.10 +[2025-03-24 13:08:47] (step=0013950) Train Loss: 3.8049, Train Steps/Sec: 1.10 +[2025-03-24 13:09:11] (step=0013975) Train Loss: 3.8479, Train Steps/Sec: 1.06 +[2025-03-24 13:09:34] (step=0014000) Train Loss: 3.9517, Train Steps/Sec: 1.10 +[2025-03-24 13:09:34] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 13:18:41] Finish Eval in 14000 steps... +[2025-03-24 13:19:05] (step=0014025) Train Loss: 3.8639, Train Steps/Sec: 0.04 +[2025-03-24 13:19:27] (step=0014050) Train Loss: 3.8946, Train Steps/Sec: 1.10 +[2025-03-24 13:19:50] (step=0014075) Train Loss: 3.8724, Train Steps/Sec: 1.10 +[2025-03-24 13:20:13] (step=0014100) Train Loss: 3.8866, Train Steps/Sec: 1.10 +[2025-03-24 13:20:36] (step=0014125) Train Loss: 3.9471, Train Steps/Sec: 1.07 +[2025-03-24 13:21:00] (step=0014150) Train Loss: 3.8986, Train Steps/Sec: 1.06 +[2025-03-24 13:21:24] (step=0014175) Train Loss: 4.0240, Train Steps/Sec: 1.05 +[2025-03-24 13:21:47] (step=0014200) Train Loss: 3.8876, Train Steps/Sec: 1.06 +[2025-03-24 13:22:10] (step=0014225) Train Loss: 3.8100, Train Steps/Sec: 1.10 +[2025-03-24 13:22:33] (step=0014250) Train Loss: 3.9576, Train Steps/Sec: 1.10 +[2025-03-24 13:22:56] (step=0014275) Train Loss: 3.8309, Train Steps/Sec: 1.10 +[2025-03-24 13:23:18] (step=0014300) Train Loss: 3.9339, Train Steps/Sec: 1.10 +[2025-03-24 13:23:41] (step=0014325) Train Loss: 3.8710, Train Steps/Sec: 1.10 +[2025-03-24 13:24:05] (step=0014350) Train Loss: 4.0023, Train Steps/Sec: 1.04 +[2025-03-24 13:24:29] (step=0014375) Train Loss: 4.0265, Train Steps/Sec: 1.05 +[2025-03-24 13:24:51] (step=0014400) Train Loss: 3.7750, Train Steps/Sec: 1.10 +[2025-03-24 13:25:14] (step=0014425) Train Loss: 4.0044, Train Steps/Sec: 1.10 +[2025-03-24 13:25:37] (step=0014450) Train Loss: 3.7259, Train Steps/Sec: 1.10 +[2025-03-24 13:26:00] (step=0014475) Train Loss: 3.8937, Train Steps/Sec: 1.10 +[2025-03-24 13:26:22] (step=0014500) Train Loss: 3.7247, Train Steps/Sec: 1.10 +[2025-03-24 13:26:45] (step=0014525) Train Loss: 3.8539, Train Steps/Sec: 1.10 +[2025-03-24 13:27:08] (step=0014550) Train Loss: 3.9759, Train Steps/Sec: 1.10 +[2025-03-24 13:27:30] (step=0014575) Train Loss: 3.8687, Train Steps/Sec: 1.10 +[2025-03-24 13:27:53] (step=0014600) Train Loss: 3.8824, Train Steps/Sec: 1.10 +[2025-03-24 13:28:16] (step=0014625) Train Loss: 3.9429, Train Steps/Sec: 1.10 +[2025-03-24 13:28:38] (step=0014650) Train Loss: 3.9459, Train Steps/Sec: 1.10 +[2025-03-24 13:29:01] (step=0014675) Train Loss: 3.9577, Train Steps/Sec: 1.10 +[2025-03-24 13:29:24] (step=0014700) Train Loss: 3.8390, Train Steps/Sec: 1.10 +[2025-03-24 13:29:46] (step=0014725) Train Loss: 3.7382, Train Steps/Sec: 1.10 +[2025-03-24 13:30:10] (step=0014750) Train Loss: 3.9281, Train Steps/Sec: 1.06 +[2025-03-24 13:30:35] (step=0014775) Train Loss: 3.9619, Train Steps/Sec: 1.01 +[2025-03-24 13:30:58] (step=0014800) Train Loss: 3.8351, Train Steps/Sec: 1.07 +[2025-03-24 13:31:21] (step=0014825) Train Loss: 3.8463, Train Steps/Sec: 1.10 +[2025-03-24 13:31:44] (step=0014850) Train Loss: 3.8982, Train Steps/Sec: 1.10 +[2025-03-24 13:32:08] (step=0014875) Train Loss: 3.9652, Train Steps/Sec: 1.05 +[2025-03-24 13:32:30] (step=0014900) Train Loss: 3.9260, Train Steps/Sec: 1.10 +[2025-03-24 13:32:54] (step=0014925) Train Loss: 4.0408, Train Steps/Sec: 1.06 +[2025-03-24 13:33:17] (step=0014950) Train Loss: 3.8804, Train Steps/Sec: 1.10 +[2025-03-24 13:33:39] (step=0014975) Train Loss: 3.8686, Train Steps/Sec: 1.10 +[2025-03-24 13:34:02] (step=0015000) Train Loss: 3.9142, Train Steps/Sec: 1.10 +[2025-03-24 13:34:25] (step=0015025) Train Loss: 4.0365, Train Steps/Sec: 1.11 +[2025-03-24 13:34:49] (step=0015050) Train Loss: 3.7829, Train Steps/Sec: 1.01 +[2025-03-24 13:35:12] (step=0015075) Train Loss: 3.9863, Train Steps/Sec: 1.10 +[2025-03-24 13:35:35] (step=0015100) Train Loss: 3.9733, Train Steps/Sec: 1.10 +[2025-03-24 13:35:57] (step=0015125) Train Loss: 3.8895, Train Steps/Sec: 1.10 +[2025-03-24 13:36:20] (step=0015150) Train Loss: 3.8821, Train Steps/Sec: 1.10 +[2025-03-24 13:36:43] (step=0015175) Train Loss: 3.8190, Train Steps/Sec: 1.10 +[2025-03-24 13:37:05] (step=0015200) Train Loss: 3.8423, Train Steps/Sec: 1.10 +[2025-03-24 13:37:28] (step=0015225) Train Loss: 3.8995, Train Steps/Sec: 1.10 +[2025-03-24 13:37:51] (step=0015250) Train Loss: 3.8992, Train Steps/Sec: 1.10 +[2025-03-24 13:38:13] (step=0015275) Train Loss: 3.8707, Train Steps/Sec: 1.10 +[2025-03-24 13:38:36] (step=0015300) Train Loss: 3.8623, Train Steps/Sec: 1.10 +[2025-03-24 13:38:59] (step=0015325) Train Loss: 3.8247, Train Steps/Sec: 1.10 +[2025-03-24 13:39:22] (step=0015350) Train Loss: 3.8844, Train Steps/Sec: 1.10 +[2025-03-24 13:39:46] (step=0015375) Train Loss: 3.8235, Train Steps/Sec: 1.05 +[2025-03-24 13:40:08] (step=0015400) Train Loss: 3.8268, Train Steps/Sec: 1.10 +[2025-03-24 13:40:31] (step=0015425) Train Loss: 3.8349, Train Steps/Sec: 1.10 +[2025-03-24 13:40:55] (step=0015450) Train Loss: 3.8962, Train Steps/Sec: 1.05 +[2025-03-24 13:41:19] (step=0015475) Train Loss: 3.7685, Train Steps/Sec: 1.06 +[2025-03-24 13:41:42] (step=0015500) Train Loss: 3.9309, Train Steps/Sec: 1.06 +[2025-03-24 13:42:05] (step=0015525) Train Loss: 3.8653, Train Steps/Sec: 1.10 +[2025-03-24 13:42:29] (step=0015550) Train Loss: 3.8114, Train Steps/Sec: 1.05 +[2025-03-24 13:42:51] (step=0015575) Train Loss: 3.8429, Train Steps/Sec: 1.10 +[2025-03-24 13:43:14] (step=0015600) Train Loss: 3.7817, Train Steps/Sec: 1.10 +[2025-03-24 13:43:38] (step=0015625) Train Loss: 3.7790, Train Steps/Sec: 1.05 +[2025-03-24 13:44:02] (step=0015650) Train Loss: 3.7889, Train Steps/Sec: 1.06 +[2025-03-24 13:44:24] (step=0015675) Train Loss: 4.0177, Train Steps/Sec: 1.10 +[2025-03-24 13:44:47] (step=0015700) Train Loss: 3.8898, Train Steps/Sec: 1.10 +[2025-03-24 13:45:10] (step=0015725) Train Loss: 3.9695, Train Steps/Sec: 1.10 +[2025-03-24 13:45:32] (step=0015750) Train Loss: 3.9651, Train Steps/Sec: 1.10 +[2025-03-24 13:45:55] (step=0015775) Train Loss: 3.7539, Train Steps/Sec: 1.10 +[2025-03-24 13:46:18] (step=0015800) Train Loss: 3.9261, Train Steps/Sec: 1.10 +[2025-03-24 13:46:40] (step=0015825) Train Loss: 3.7777, Train Steps/Sec: 1.10 +[2025-03-24 13:47:04] (step=0015850) Train Loss: 3.9041, Train Steps/Sec: 1.07 +[2025-03-24 13:47:27] (step=0015875) Train Loss: 3.8618, Train Steps/Sec: 1.10 +[2025-03-24 13:47:49] (step=0015900) Train Loss: 3.8864, Train Steps/Sec: 1.10 +[2025-03-24 13:48:12] (step=0015925) Train Loss: 3.8504, Train Steps/Sec: 1.10 +[2025-03-24 13:48:35] (step=0015950) Train Loss: 3.9155, Train Steps/Sec: 1.10 +[2025-03-24 13:48:58] (step=0015975) Train Loss: 3.9521, Train Steps/Sec: 1.10 +[2025-03-24 13:49:20] (step=0016000) Train Loss: 3.9791, Train Steps/Sec: 1.10 +[2025-03-24 13:49:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 13:58:24] Finish Eval in 16000 steps... +[2025-03-24 13:58:43] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0016000.pt +[2025-03-24 13:58:45] Removed old checkpoint: checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0012000.pt +[2025-03-24 13:59:08] (step=0016025) Train Loss: 3.8066, Train Steps/Sec: 0.04 +[2025-03-24 13:59:31] (step=0016050) Train Loss: 3.9557, Train Steps/Sec: 1.10 +[2025-03-24 13:59:54] (step=0016075) Train Loss: 3.9360, Train Steps/Sec: 1.10 +[2025-03-24 14:00:16] (step=0016100) Train Loss: 3.9689, Train Steps/Sec: 1.10 +[2025-03-24 14:00:39] (step=0016125) Train Loss: 3.7709, Train Steps/Sec: 1.10 +[2025-03-24 14:01:03] (step=0016150) Train Loss: 3.7712, Train Steps/Sec: 1.06 +[2025-03-24 14:01:29] (step=0016175) Train Loss: 3.8381, Train Steps/Sec: 0.97 +[2025-03-24 14:01:51] (step=0016200) Train Loss: 3.8382, Train Steps/Sec: 1.09 +[2025-03-24 14:02:15] (step=0016225) Train Loss: 3.9317, Train Steps/Sec: 1.05 +[2025-03-24 14:02:39] (step=0016250) Train Loss: 3.8412, Train Steps/Sec: 1.06 +[2025-03-24 14:03:01] (step=0016275) Train Loss: 3.8575, Train Steps/Sec: 1.10 +[2025-03-24 14:03:24] (step=0016300) Train Loss: 3.8671, Train Steps/Sec: 1.10 +[2025-03-24 14:03:47] (step=0016325) Train Loss: 3.8698, Train Steps/Sec: 1.10 +[2025-03-24 14:04:09] (step=0016350) Train Loss: 3.8967, Train Steps/Sec: 1.10 +[2025-03-24 14:04:32] (step=0016375) Train Loss: 3.8295, Train Steps/Sec: 1.10 +[2025-03-24 14:04:55] (step=0016400) Train Loss: 3.9397, Train Steps/Sec: 1.10 +[2025-03-24 14:05:17] (step=0016425) Train Loss: 3.8931, Train Steps/Sec: 1.10 +[2025-03-24 14:05:41] (step=0016450) Train Loss: 3.8986, Train Steps/Sec: 1.07 +[2025-03-24 14:06:03] (step=0016475) Train Loss: 3.8616, Train Steps/Sec: 1.10 +[2025-03-24 14:06:26] (step=0016500) Train Loss: 3.8681, Train Steps/Sec: 1.10 +[2025-03-24 14:06:49] (step=0016525) Train Loss: 3.7683, Train Steps/Sec: 1.10 +[2025-03-24 14:07:12] (step=0016550) Train Loss: 3.8334, Train Steps/Sec: 1.10 +[2025-03-24 14:07:35] (step=0016575) Train Loss: 3.8202, Train Steps/Sec: 1.05 +[2025-03-24 14:07:59] (step=0016600) Train Loss: 3.8610, Train Steps/Sec: 1.05 +[2025-03-24 14:08:22] (step=0016625) Train Loss: 3.9381, Train Steps/Sec: 1.10 +[2025-03-24 14:08:45] (step=0016650) Train Loss: 3.8912, Train Steps/Sec: 1.10 +[2025-03-24 14:09:07] (step=0016675) Train Loss: 3.7935, Train Steps/Sec: 1.10 +[2025-03-24 14:09:30] (step=0016700) Train Loss: 3.9022, Train Steps/Sec: 1.10 +[2025-03-24 14:09:54] (step=0016725) Train Loss: 3.7605, Train Steps/Sec: 1.05 +[2025-03-24 14:10:17] (step=0016750) Train Loss: 3.7745, Train Steps/Sec: 1.10 +[2025-03-24 14:10:39] (step=0016775) Train Loss: 3.9360, Train Steps/Sec: 1.10 +[2025-03-24 14:11:02] (step=0016800) Train Loss: 3.8303, Train Steps/Sec: 1.10 +[2025-03-24 14:11:25] (step=0016825) Train Loss: 3.8536, Train Steps/Sec: 1.10 +[2025-03-24 14:11:48] (step=0016850) Train Loss: 3.7777, Train Steps/Sec: 1.07 +[2025-03-24 14:12:11] (step=0016875) Train Loss: 3.8617, Train Steps/Sec: 1.10 +[2025-03-24 14:12:34] (step=0016900) Train Loss: 3.9413, Train Steps/Sec: 1.10 +[2025-03-24 14:12:56] (step=0016925) Train Loss: 3.8253, Train Steps/Sec: 1.10 +[2025-03-24 14:13:19] (step=0016950) Train Loss: 3.8193, Train Steps/Sec: 1.10 +[2025-03-24 14:13:43] (step=0016975) Train Loss: 3.9212, Train Steps/Sec: 1.06 +[2025-03-24 14:14:07] (step=0017000) Train Loss: 3.7602, Train Steps/Sec: 1.02 +[2025-03-24 14:14:31] (step=0017025) Train Loss: 3.9568, Train Steps/Sec: 1.05 +[2025-03-24 14:14:54] (step=0017050) Train Loss: 3.8706, Train Steps/Sec: 1.10 +[2025-03-24 14:15:16] (step=0017075) Train Loss: 3.8716, Train Steps/Sec: 1.10 +[2025-03-24 14:15:39] (step=0017100) Train Loss: 3.9352, Train Steps/Sec: 1.10 +[2025-03-24 14:16:02] (step=0017125) Train Loss: 3.8118, Train Steps/Sec: 1.10 +[2025-03-24 14:16:25] (step=0017150) Train Loss: 3.8913, Train Steps/Sec: 1.10 +[2025-03-24 14:16:48] (step=0017175) Train Loss: 3.7869, Train Steps/Sec: 1.05 +[2025-03-24 14:17:11] (step=0017200) Train Loss: 3.8789, Train Steps/Sec: 1.10 +[2025-03-24 14:17:35] (step=0017225) Train Loss: 3.9886, Train Steps/Sec: 1.06 +[2025-03-24 14:17:57] (step=0017250) Train Loss: 3.8319, Train Steps/Sec: 1.10 +[2025-03-24 14:18:20] (step=0017275) Train Loss: 3.7174, Train Steps/Sec: 1.10 +[2025-03-24 14:18:43] (step=0017300) Train Loss: 3.8653, Train Steps/Sec: 1.10 +[2025-03-24 14:19:06] (step=0017325) Train Loss: 3.8260, Train Steps/Sec: 1.10 +[2025-03-24 14:19:28] (step=0017350) Train Loss: 3.9005, Train Steps/Sec: 1.10 +[2025-03-24 14:19:51] (step=0017375) Train Loss: 3.8592, Train Steps/Sec: 1.10 +[2025-03-24 14:20:14] (step=0017400) Train Loss: 3.8639, Train Steps/Sec: 1.10 +[2025-03-24 14:20:37] (step=0017425) Train Loss: 3.9175, Train Steps/Sec: 1.10 +[2025-03-24 14:20:59] (step=0017450) Train Loss: 3.8503, Train Steps/Sec: 1.10 +[2025-03-24 14:21:22] (step=0017475) Train Loss: 3.8219, Train Steps/Sec: 1.10 +[2025-03-24 14:21:46] (step=0017500) Train Loss: 3.7731, Train Steps/Sec: 1.05 +[2025-03-24 14:22:11] (step=0017525) Train Loss: 3.7650, Train Steps/Sec: 1.01 +[2025-03-24 14:22:33] (step=0017550) Train Loss: 3.9273, Train Steps/Sec: 1.10 +[2025-03-24 14:22:56] (step=0017575) Train Loss: 3.9033, Train Steps/Sec: 1.10 +[2025-03-24 14:23:19] (step=0017600) Train Loss: 3.8509, Train Steps/Sec: 1.10 +[2025-03-24 14:23:41] (step=0017625) Train Loss: 3.8548, Train Steps/Sec: 1.10 +[2025-03-24 14:24:04] (step=0017650) Train Loss: 3.8728, Train Steps/Sec: 1.10 +[2025-03-24 14:24:27] (step=0017675) Train Loss: 3.8869, Train Steps/Sec: 1.10 +[2025-03-24 14:24:52] (step=0017700) Train Loss: 3.8947, Train Steps/Sec: 1.00 +[2025-03-24 14:25:15] (step=0017725) Train Loss: 3.8783, Train Steps/Sec: 1.06 +[2025-03-24 14:25:39] (step=0017750) Train Loss: 3.8678, Train Steps/Sec: 1.06 +[2025-03-24 14:26:03] (step=0017775) Train Loss: 3.9771, Train Steps/Sec: 1.06 +[2025-03-24 14:26:25] (step=0017800) Train Loss: 3.8506, Train Steps/Sec: 1.10 +[2025-03-24 14:26:48] (step=0017825) Train Loss: 3.7639, Train Steps/Sec: 1.10 +[2025-03-24 14:27:11] (step=0017850) Train Loss: 3.8348, Train Steps/Sec: 1.10 +[2025-03-24 14:27:33] (step=0017875) Train Loss: 3.6688, Train Steps/Sec: 1.10 +[2025-03-24 14:27:56] (step=0017900) Train Loss: 3.8955, Train Steps/Sec: 1.10 +[2025-03-24 14:28:19] (step=0017925) Train Loss: 3.8874, Train Steps/Sec: 1.10 +[2025-03-24 14:28:41] (step=0017950) Train Loss: 3.8180, Train Steps/Sec: 1.10 +[2025-03-24 14:29:05] (step=0017975) Train Loss: 3.8059, Train Steps/Sec: 1.05 +[2025-03-24 14:29:28] (step=0018000) Train Loss: 3.8513, Train Steps/Sec: 1.10 +[2025-03-24 14:29:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 14:38:35] Finish Eval in 18000 steps... +[2025-03-24 14:38:58] (step=0018025) Train Loss: 3.7996, Train Steps/Sec: 0.04 +[2025-03-24 14:39:21] (step=0018050) Train Loss: 3.8961, Train Steps/Sec: 1.10 +[2025-03-24 14:39:44] (step=0018075) Train Loss: 3.7416, Train Steps/Sec: 1.09 +[2025-03-24 14:40:06] (step=0018100) Train Loss: 3.7171, Train Steps/Sec: 1.10 +[2025-03-24 14:40:29] (step=0018125) Train Loss: 3.8075, Train Steps/Sec: 1.10 +[2025-03-24 14:40:52] (step=0018150) Train Loss: 3.9347, Train Steps/Sec: 1.10 +[2025-03-24 14:41:15] (step=0018175) Train Loss: 3.8404, Train Steps/Sec: 1.10 +[2025-03-24 14:41:37] (step=0018200) Train Loss: 3.8659, Train Steps/Sec: 1.10 +[2025-03-24 14:42:01] (step=0018225) Train Loss: 3.7767, Train Steps/Sec: 1.07 +[2025-03-24 14:42:23] (step=0018250) Train Loss: 3.7942, Train Steps/Sec: 1.10 +[2025-03-24 14:42:47] (step=0018275) Train Loss: 3.7771, Train Steps/Sec: 1.06 +[2025-03-24 14:43:10] (step=0018300) Train Loss: 3.8449, Train Steps/Sec: 1.10 +[2025-03-24 14:43:33] (step=0018325) Train Loss: 3.8579, Train Steps/Sec: 1.05 +[2025-03-24 14:43:56] (step=0018350) Train Loss: 3.8469, Train Steps/Sec: 1.10 +[2025-03-24 14:44:19] (step=0018375) Train Loss: 3.8078, Train Steps/Sec: 1.10 +[2025-03-24 14:44:43] (step=0018400) Train Loss: 3.7304, Train Steps/Sec: 1.05 +[2025-03-24 14:45:06] (step=0018425) Train Loss: 3.7395, Train Steps/Sec: 1.10 +[2025-03-24 14:45:29] (step=0018450) Train Loss: 3.9243, Train Steps/Sec: 1.10 +[2025-03-24 14:45:52] (step=0018475) Train Loss: 3.8670, Train Steps/Sec: 1.11 +[2025-03-24 14:46:14] (step=0018500) Train Loss: 3.8429, Train Steps/Sec: 1.10 +[2025-03-24 14:46:37] (step=0018525) Train Loss: 3.9065, Train Steps/Sec: 1.10 +[2025-03-24 14:47:01] (step=0018550) Train Loss: 3.9229, Train Steps/Sec: 1.06 +[2025-03-24 14:47:24] (step=0018575) Train Loss: 3.7993, Train Steps/Sec: 1.05 +[2025-03-24 14:47:47] (step=0018600) Train Loss: 3.7361, Train Steps/Sec: 1.10 +[2025-03-24 14:48:10] (step=0018625) Train Loss: 3.7571, Train Steps/Sec: 1.10 +[2025-03-24 14:48:33] (step=0018650) Train Loss: 3.7463, Train Steps/Sec: 1.10 +[2025-03-24 14:48:55] (step=0018675) Train Loss: 3.7820, Train Steps/Sec: 1.10 +[2025-03-24 14:49:19] (step=0018700) Train Loss: 3.8422, Train Steps/Sec: 1.04 +[2025-03-24 14:49:42] (step=0018725) Train Loss: 3.8843, Train Steps/Sec: 1.10 +[2025-03-24 14:50:05] (step=0018750) Train Loss: 3.8065, Train Steps/Sec: 1.10 +[2025-03-24 14:50:27] (step=0018775) Train Loss: 3.8358, Train Steps/Sec: 1.10 +[2025-03-24 14:50:50] (step=0018800) Train Loss: 3.7009, Train Steps/Sec: 1.10 +[2025-03-24 14:51:13] (step=0018825) Train Loss: 3.7547, Train Steps/Sec: 1.10 +[2025-03-24 14:51:36] (step=0018850) Train Loss: 3.8871, Train Steps/Sec: 1.10 +[2025-03-24 14:51:58] (step=0018875) Train Loss: 3.7758, Train Steps/Sec: 1.10 +[2025-03-24 14:52:21] (step=0018900) Train Loss: 3.8429, Train Steps/Sec: 1.10 +[2025-03-24 14:52:46] (step=0018925) Train Loss: 3.8349, Train Steps/Sec: 1.03 +[2025-03-24 14:53:09] (step=0018950) Train Loss: 3.8328, Train Steps/Sec: 1.10 +[2025-03-24 14:53:33] (step=0018975) Train Loss: 3.8681, Train Steps/Sec: 1.06 +[2025-03-24 14:53:55] (step=0019000) Train Loss: 3.7820, Train Steps/Sec: 1.10 +[2025-03-24 14:54:18] (step=0019025) Train Loss: 3.8375, Train Steps/Sec: 1.10 +[2025-03-24 14:54:41] (step=0019050) Train Loss: 3.8439, Train Steps/Sec: 1.10 +[2025-03-24 14:55:04] (step=0019075) Train Loss: 3.7178, Train Steps/Sec: 1.10 +[2025-03-24 14:55:26] (step=0019100) Train Loss: 3.7700, Train Steps/Sec: 1.10 +[2025-03-24 14:55:50] (step=0019125) Train Loss: 3.8169, Train Steps/Sec: 1.05 +[2025-03-24 14:56:14] (step=0019150) Train Loss: 3.8558, Train Steps/Sec: 1.05 +[2025-03-24 14:56:37] (step=0019175) Train Loss: 3.9177, Train Steps/Sec: 1.06 +[2025-03-24 14:57:00] (step=0019200) Train Loss: 3.7491, Train Steps/Sec: 1.10 +[2025-03-24 14:57:23] (step=0019225) Train Loss: 3.8615, Train Steps/Sec: 1.10 +[2025-03-24 14:57:45] (step=0019250) Train Loss: 3.9140, Train Steps/Sec: 1.10 +[2025-03-24 14:58:08] (step=0019275) Train Loss: 3.8395, Train Steps/Sec: 1.10 +[2025-03-24 14:58:31] (step=0019300) Train Loss: 3.8726, Train Steps/Sec: 1.10 +[2025-03-24 14:58:53] (step=0019325) Train Loss: 3.8306, Train Steps/Sec: 1.10 +[2025-03-24 14:59:16] (step=0019350) Train Loss: 3.7320, Train Steps/Sec: 1.10 +[2025-03-24 14:59:39] (step=0019375) Train Loss: 3.7602, Train Steps/Sec: 1.10 +[2025-03-24 15:00:03] (step=0019400) Train Loss: 3.8010, Train Steps/Sec: 1.02 +[2025-03-24 15:00:26] (step=0019425) Train Loss: 3.7561, Train Steps/Sec: 1.10 +[2025-03-24 15:00:50] (step=0019450) Train Loss: 3.9424, Train Steps/Sec: 1.05 +[2025-03-24 15:01:13] (step=0019475) Train Loss: 3.7782, Train Steps/Sec: 1.10 +[2025-03-24 15:01:35] (step=0019500) Train Loss: 3.7318, Train Steps/Sec: 1.10 +[2025-03-24 15:01:59] (step=0019525) Train Loss: 3.9102, Train Steps/Sec: 1.05 +[2025-03-24 15:02:22] (step=0019550) Train Loss: 3.8321, Train Steps/Sec: 1.10 +[2025-03-24 15:02:45] (step=0019575) Train Loss: 3.7027, Train Steps/Sec: 1.10 +[2025-03-24 15:03:07] (step=0019600) Train Loss: 3.7694, Train Steps/Sec: 1.10 +[2025-03-24 15:03:30] (step=0019625) Train Loss: 3.7614, Train Steps/Sec: 1.10 +[2025-03-24 15:03:53] (step=0019650) Train Loss: 3.7758, Train Steps/Sec: 1.10 +[2025-03-24 15:04:15] (step=0019675) Train Loss: 3.7746, Train Steps/Sec: 1.10 +[2025-03-24 15:04:38] (step=0019700) Train Loss: 3.8159, Train Steps/Sec: 1.10 +[2025-03-24 15:05:02] (step=0019725) Train Loss: 3.7827, Train Steps/Sec: 1.06 +[2025-03-24 15:05:24] (step=0019750) Train Loss: 3.7938, Train Steps/Sec: 1.10 +[2025-03-24 15:05:47] (step=0019775) Train Loss: 3.7209, Train Steps/Sec: 1.10 +[2025-03-24 15:06:11] (step=0019800) Train Loss: 3.8526, Train Steps/Sec: 1.06 +[2025-03-24 15:06:35] (step=0019825) Train Loss: 3.8777, Train Steps/Sec: 1.06 +[2025-03-24 15:06:59] (step=0019850) Train Loss: 3.7975, Train Steps/Sec: 1.03 +[2025-03-24 15:07:21] (step=0019875) Train Loss: 3.9378, Train Steps/Sec: 1.10 +[2025-03-24 15:07:44] (step=0019900) Train Loss: 3.8433, Train Steps/Sec: 1.10 +[2025-03-24 15:08:07] (step=0019925) Train Loss: 3.8253, Train Steps/Sec: 1.10 +[2025-03-24 15:08:31] (step=0019950) Train Loss: 3.7303, Train Steps/Sec: 1.04 +[2025-03-24 15:08:54] (step=0019975) Train Loss: 3.8406, Train Steps/Sec: 1.10 +[2025-03-24 15:09:16] (step=0020000) Train Loss: 3.7719, Train Steps/Sec: 1.10 +[2025-03-24 15:09:16] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 15:18:21] Finish Eval in 20000 steps... +[2025-03-24 15:18:40] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0020000.pt +[2025-03-24 15:18:42] Removed old checkpoint: checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0016000.pt +[2025-03-24 15:19:05] (step=0020025) Train Loss: 3.8250, Train Steps/Sec: 0.04 +[2025-03-24 15:19:28] (step=0020050) Train Loss: 3.8167, Train Steps/Sec: 1.10 +[2025-03-24 15:19:51] (step=0020075) Train Loss: 3.8311, Train Steps/Sec: 1.10 +[2025-03-24 15:20:13] (step=0020100) Train Loss: 3.7652, Train Steps/Sec: 1.10 +[2025-03-24 15:20:36] (step=0020125) Train Loss: 3.7086, Train Steps/Sec: 1.10 +[2025-03-24 15:21:00] (step=0020150) Train Loss: 3.7857, Train Steps/Sec: 1.06 +[2025-03-24 15:21:22] (step=0020175) Train Loss: 3.7811, Train Steps/Sec: 1.10 +[2025-03-24 15:21:45] (step=0020200) Train Loss: 3.9040, Train Steps/Sec: 1.10 +[2025-03-24 15:22:08] (step=0020225) Train Loss: 3.8397, Train Steps/Sec: 1.10 +[2025-03-24 15:22:31] (step=0020250) Train Loss: 3.6060, Train Steps/Sec: 1.10 +[2025-03-24 15:22:54] (step=0020275) Train Loss: 3.7767, Train Steps/Sec: 1.10 +[2025-03-24 15:23:16] (step=0020300) Train Loss: 3.8717, Train Steps/Sec: 1.10 +[2025-03-24 15:23:39] (step=0020325) Train Loss: 3.6832, Train Steps/Sec: 1.10 +[2025-03-24 15:24:02] (step=0020350) Train Loss: 3.8702, Train Steps/Sec: 1.10 +[2025-03-24 15:24:25] (step=0020375) Train Loss: 3.9500, Train Steps/Sec: 1.10 +[2025-03-24 15:24:48] (step=0020400) Train Loss: 3.8108, Train Steps/Sec: 1.05 +[2025-03-24 15:25:11] (step=0020425) Train Loss: 3.7464, Train Steps/Sec: 1.10 +[2025-03-24 15:25:34] (step=0020450) Train Loss: 3.8610, Train Steps/Sec: 1.10 +[2025-03-24 15:25:59] (step=0020475) Train Loss: 3.7623, Train Steps/Sec: 1.00 +[2025-03-24 15:26:23] (step=0020500) Train Loss: 3.8346, Train Steps/Sec: 1.05 +[2025-03-24 15:26:46] (step=0020525) Train Loss: 3.9061, Train Steps/Sec: 1.07 +[2025-03-24 15:27:09] (step=0020550) Train Loss: 3.7506, Train Steps/Sec: 1.10 +[2025-03-24 15:27:33] (step=0020575) Train Loss: 3.7661, Train Steps/Sec: 1.02 +[2025-03-24 15:27:56] (step=0020600) Train Loss: 3.8146, Train Steps/Sec: 1.10 +[2025-03-24 15:28:19] (step=0020625) Train Loss: 3.8336, Train Steps/Sec: 1.10 +[2025-03-24 15:28:41] (step=0020650) Train Loss: 3.9079, Train Steps/Sec: 1.10 +[2025-03-24 15:29:04] (step=0020675) Train Loss: 3.7989, Train Steps/Sec: 1.10 +[2025-03-24 15:29:27] (step=0020700) Train Loss: 3.7889, Train Steps/Sec: 1.10 +[2025-03-24 15:29:50] (step=0020725) Train Loss: 3.8150, Train Steps/Sec: 1.10 +[2025-03-24 15:30:12] (step=0020750) Train Loss: 3.8964, Train Steps/Sec: 1.10 +[2025-03-24 15:30:35] (step=0020775) Train Loss: 3.7896, Train Steps/Sec: 1.10 +[2025-03-24 15:30:58] (step=0020800) Train Loss: 3.8659, Train Steps/Sec: 1.10 +[2025-03-24 15:31:22] (step=0020825) Train Loss: 3.8461, Train Steps/Sec: 1.05 +[2025-03-24 15:31:44] (step=0020850) Train Loss: 3.9097, Train Steps/Sec: 1.10 +[2025-03-24 15:32:07] (step=0020875) Train Loss: 3.7809, Train Steps/Sec: 1.10 +[2025-03-24 15:32:30] (step=0020900) Train Loss: 3.8699, Train Steps/Sec: 1.07 +[2025-03-24 15:32:54] (step=0020925) Train Loss: 3.8049, Train Steps/Sec: 1.05 +[2025-03-24 15:33:17] (step=0020950) Train Loss: 3.8038, Train Steps/Sec: 1.10 +[2025-03-24 15:33:40] (step=0020975) Train Loss: 3.7395, Train Steps/Sec: 1.10 +[2025-03-24 15:34:03] (step=0021000) Train Loss: 3.6907, Train Steps/Sec: 1.10 +[2025-03-24 15:34:25] (step=0021025) Train Loss: 3.8140, Train Steps/Sec: 1.10 +[2025-03-24 15:34:49] (step=0021050) Train Loss: 3.8399, Train Steps/Sec: 1.06 +[2025-03-24 15:35:12] (step=0021075) Train Loss: 3.7291, Train Steps/Sec: 1.10 +[2025-03-24 15:35:34] (step=0021100) Train Loss: 3.8885, Train Steps/Sec: 1.10 +[2025-03-24 15:35:57] (step=0021125) Train Loss: 3.8098, Train Steps/Sec: 1.10 +[2025-03-24 15:36:22] (step=0021150) Train Loss: 3.8700, Train Steps/Sec: 1.00 +[2025-03-24 15:36:45] (step=0021175) Train Loss: 3.8682, Train Steps/Sec: 1.10 +[2025-03-24 15:37:08] (step=0021200) Train Loss: 3.6633, Train Steps/Sec: 1.05 +[2025-03-24 15:37:31] (step=0021225) Train Loss: 3.8380, Train Steps/Sec: 1.10 +[2025-03-24 15:37:54] (step=0021250) Train Loss: 3.7485, Train Steps/Sec: 1.10 +[2025-03-24 15:38:17] (step=0021275) Train Loss: 3.9122, Train Steps/Sec: 1.10 +[2025-03-24 15:38:39] (step=0021300) Train Loss: 3.8057, Train Steps/Sec: 1.10 +[2025-03-24 15:39:02] (step=0021325) Train Loss: 3.7669, Train Steps/Sec: 1.10 +[2025-03-24 15:39:25] (step=0021350) Train Loss: 3.8626, Train Steps/Sec: 1.10 +[2025-03-24 15:39:48] (step=0021375) Train Loss: 3.8623, Train Steps/Sec: 1.10 +[2025-03-24 15:40:10] (step=0021400) Train Loss: 3.8982, Train Steps/Sec: 1.10 +[2025-03-24 15:40:34] (step=0021425) Train Loss: 3.6369, Train Steps/Sec: 1.06 +[2025-03-24 15:40:57] (step=0021450) Train Loss: 3.9659, Train Steps/Sec: 1.10 +[2025-03-24 15:41:19] (step=0021475) Train Loss: 3.6314, Train Steps/Sec: 1.10 +[2025-03-24 15:41:42] (step=0021500) Train Loss: 3.7069, Train Steps/Sec: 1.10 +[2025-03-24 15:42:05] (step=0021525) Train Loss: 3.8385, Train Steps/Sec: 1.10 +[2025-03-24 15:42:30] (step=0021550) Train Loss: 3.7781, Train Steps/Sec: 1.00 +[2025-03-24 15:42:53] (step=0021575) Train Loss: 3.7920, Train Steps/Sec: 1.07 +[2025-03-24 15:43:16] (step=0021600) Train Loss: 3.7023, Train Steps/Sec: 1.10 +[2025-03-24 15:43:39] (step=0021625) Train Loss: 3.8334, Train Steps/Sec: 1.06 +[2025-03-24 15:44:02] (step=0021650) Train Loss: 3.9187, Train Steps/Sec: 1.10 +[2025-03-24 15:44:25] (step=0021675) Train Loss: 3.7860, Train Steps/Sec: 1.10 +[2025-03-24 15:44:48] (step=0021700) Train Loss: 3.9311, Train Steps/Sec: 1.10 +[2025-03-24 15:45:10] (step=0021725) Train Loss: 3.7109, Train Steps/Sec: 1.10 +[2025-03-24 15:45:33] (step=0021750) Train Loss: 3.9228, Train Steps/Sec: 1.10 +[2025-03-24 15:45:56] (step=0021775) Train Loss: 3.8376, Train Steps/Sec: 1.10 +[2025-03-24 15:46:19] (step=0021800) Train Loss: 3.8201, Train Steps/Sec: 1.06 +[2025-03-24 15:46:42] (step=0021825) Train Loss: 3.8652, Train Steps/Sec: 1.10 +[2025-03-24 15:47:06] (step=0021850) Train Loss: 3.9279, Train Steps/Sec: 1.05 +[2025-03-24 15:47:29] (step=0021875) Train Loss: 3.7959, Train Steps/Sec: 1.10 +[2025-03-24 15:47:54] (step=0021900) Train Loss: 3.7760, Train Steps/Sec: 1.05 +[2025-03-24 15:48:16] (step=0021925) Train Loss: 3.7868, Train Steps/Sec: 1.10 +[2025-03-24 15:48:39] (step=0021950) Train Loss: 3.7188, Train Steps/Sec: 1.10 +[2025-03-24 15:49:02] (step=0021975) Train Loss: 3.9292, Train Steps/Sec: 1.10 +[2025-03-24 15:49:25] (step=0022000) Train Loss: 3.7860, Train Steps/Sec: 1.10 +[2025-03-24 15:49:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 15:58:30] Finish Eval in 22000 steps... +[2025-03-24 15:58:54] (step=0022025) Train Loss: 3.7337, Train Steps/Sec: 0.04 +[2025-03-24 15:59:16] (step=0022050) Train Loss: 3.7349, Train Steps/Sec: 1.10 +[2025-03-24 15:59:40] (step=0022075) Train Loss: 3.8537, Train Steps/Sec: 1.07 +[2025-03-24 16:00:03] (step=0022100) Train Loss: 3.7463, Train Steps/Sec: 1.10 +[2025-03-24 16:00:26] (step=0022125) Train Loss: 3.7548, Train Steps/Sec: 1.05 +[2025-03-24 16:00:49] (step=0022150) Train Loss: 3.6521, Train Steps/Sec: 1.10 +[2025-03-24 16:01:12] (step=0022175) Train Loss: 3.8797, Train Steps/Sec: 1.10 +[2025-03-24 16:01:36] (step=0022200) Train Loss: 3.8241, Train Steps/Sec: 1.05 +[2025-03-24 16:01:58] (step=0022225) Train Loss: 3.7901, Train Steps/Sec: 1.10 +[2025-03-24 16:02:21] (step=0022250) Train Loss: 3.7686, Train Steps/Sec: 1.10 +[2025-03-24 16:02:44] (step=0022275) Train Loss: 3.9818, Train Steps/Sec: 1.10 +[2025-03-24 16:03:06] (step=0022300) Train Loss: 3.7481, Train Steps/Sec: 1.10 +[2025-03-24 16:03:29] (step=0022325) Train Loss: 3.8330, Train Steps/Sec: 1.10 +[2025-03-24 16:03:52] (step=0022350) Train Loss: 3.8599, Train Steps/Sec: 1.10 +[2025-03-24 16:04:16] (step=0022375) Train Loss: 3.8269, Train Steps/Sec: 1.05 +[2025-03-24 16:04:38] (step=0022400) Train Loss: 3.6981, Train Steps/Sec: 1.10 +[2025-03-24 16:05:02] (step=0022425) Train Loss: 3.6829, Train Steps/Sec: 1.06 +[2025-03-24 16:05:25] (step=0022450) Train Loss: 4.0023, Train Steps/Sec: 1.09 +[2025-03-24 16:05:47] (step=0022475) Train Loss: 3.7457, Train Steps/Sec: 1.10 +[2025-03-24 16:06:10] (step=0022500) Train Loss: 3.8221, Train Steps/Sec: 1.10 +[2025-03-24 16:06:33] (step=0022525) Train Loss: 3.7872, Train Steps/Sec: 1.10 +[2025-03-24 16:06:56] (step=0022550) Train Loss: 3.8212, Train Steps/Sec: 1.10 +[2025-03-24 16:07:19] (step=0022575) Train Loss: 3.8870, Train Steps/Sec: 1.07 +[2025-03-24 16:07:43] (step=0022600) Train Loss: 3.7920, Train Steps/Sec: 1.05 +[2025-03-24 16:08:06] (step=0022625) Train Loss: 3.6706, Train Steps/Sec: 1.10 +[2025-03-24 16:08:31] (step=0022650) Train Loss: 3.7876, Train Steps/Sec: 1.00 +[2025-03-24 16:08:53] (step=0022675) Train Loss: 3.6984, Train Steps/Sec: 1.10 +[2025-03-24 16:09:17] (step=0022700) Train Loss: 3.7029, Train Steps/Sec: 1.05 +[2025-03-24 16:09:40] (step=0022725) Train Loss: 3.7628, Train Steps/Sec: 1.10 +[2025-03-24 16:10:02] (step=0022750) Train Loss: 3.7639, Train Steps/Sec: 1.10 +[2025-03-24 16:10:25] (step=0022775) Train Loss: 3.8560, Train Steps/Sec: 1.10 +[2025-03-24 16:10:48] (step=0022800) Train Loss: 3.8734, Train Steps/Sec: 1.10 +[2025-03-24 16:11:11] (step=0022825) Train Loss: 3.6483, Train Steps/Sec: 1.10 +[2025-03-24 16:11:34] (step=0022850) Train Loss: 3.7839, Train Steps/Sec: 1.05 +[2025-03-24 16:11:57] (step=0022875) Train Loss: 3.7585, Train Steps/Sec: 1.10 +[2025-03-24 16:12:20] (step=0022900) Train Loss: 3.7584, Train Steps/Sec: 1.10 +[2025-03-24 16:12:43] (step=0022925) Train Loss: 3.8295, Train Steps/Sec: 1.05 +[2025-03-24 16:13:06] (step=0022950) Train Loss: 3.7967, Train Steps/Sec: 1.10 +[2025-03-24 16:13:30] (step=0022975) Train Loss: 3.7504, Train Steps/Sec: 1.06 +[2025-03-24 16:13:52] (step=0023000) Train Loss: 3.8730, Train Steps/Sec: 1.10 +[2025-03-24 16:14:15] (step=0023025) Train Loss: 3.7314, Train Steps/Sec: 1.10 +[2025-03-24 16:14:38] (step=0023050) Train Loss: 3.6846, Train Steps/Sec: 1.10 +[2025-03-24 16:15:01] (step=0023075) Train Loss: 3.8173, Train Steps/Sec: 1.10 +[2025-03-24 16:15:25] (step=0023100) Train Loss: 3.7309, Train Steps/Sec: 1.05 +[2025-03-24 16:15:47] (step=0023125) Train Loss: 3.8529, Train Steps/Sec: 1.10 +[2025-03-24 16:16:10] (step=0023150) Train Loss: 3.8705, Train Steps/Sec: 1.10 +[2025-03-24 16:16:33] (step=0023175) Train Loss: 3.7021, Train Steps/Sec: 1.10 +[2025-03-24 16:16:56] (step=0023200) Train Loss: 3.7131, Train Steps/Sec: 1.10 +[2025-03-24 16:17:18] (step=0023225) Train Loss: 3.8292, Train Steps/Sec: 1.10 +[2025-03-24 16:17:42] (step=0023250) Train Loss: 3.8947, Train Steps/Sec: 1.07 +[2025-03-24 16:18:04] (step=0023275) Train Loss: 3.7813, Train Steps/Sec: 1.10 +[2025-03-24 16:18:28] (step=0023300) Train Loss: 3.7857, Train Steps/Sec: 1.06 +[2025-03-24 16:18:52] (step=0023325) Train Loss: 3.8995, Train Steps/Sec: 1.06 +[2025-03-24 16:19:14] (step=0023350) Train Loss: 3.7723, Train Steps/Sec: 1.10 +[2025-03-24 16:19:37] (step=0023375) Train Loss: 3.7576, Train Steps/Sec: 1.10 +[2025-03-24 16:20:00] (step=0023400) Train Loss: 3.7430, Train Steps/Sec: 1.10 +[2025-03-24 16:20:24] (step=0023425) Train Loss: 3.9693, Train Steps/Sec: 1.04 +[2025-03-24 16:20:47] (step=0023450) Train Loss: 3.8228, Train Steps/Sec: 1.10 +[2025-03-24 16:21:09] (step=0023475) Train Loss: 3.8947, Train Steps/Sec: 1.10 +[2025-03-24 16:21:32] (step=0023500) Train Loss: 3.8286, Train Steps/Sec: 1.10 +[2025-03-24 16:21:55] (step=0023525) Train Loss: 3.8494, Train Steps/Sec: 1.10 +[2025-03-24 16:22:17] (step=0023550) Train Loss: 3.7075, Train Steps/Sec: 1.10 +[2025-03-24 16:22:40] (step=0023575) Train Loss: 3.7215, Train Steps/Sec: 1.10 +[2025-03-24 16:23:03] (step=0023600) Train Loss: 3.7512, Train Steps/Sec: 1.10 +[2025-03-24 16:23:25] (step=0023625) Train Loss: 3.6911, Train Steps/Sec: 1.10 +[2025-03-24 16:23:48] (step=0023650) Train Loss: 3.6616, Train Steps/Sec: 1.10 +[2025-03-24 16:24:11] (step=0023675) Train Loss: 3.7406, Train Steps/Sec: 1.10 +[2025-03-24 16:24:35] (step=0023700) Train Loss: 3.7819, Train Steps/Sec: 1.05 +[2025-03-24 16:24:58] (step=0023725) Train Loss: 3.8042, Train Steps/Sec: 1.05 +[2025-03-24 16:25:22] (step=0023750) Train Loss: 3.8237, Train Steps/Sec: 1.06 +[2025-03-24 16:25:46] (step=0023775) Train Loss: 3.8001, Train Steps/Sec: 1.05 +[2025-03-24 16:26:09] (step=0023800) Train Loss: 3.9019, Train Steps/Sec: 1.10 +[2025-03-24 16:26:31] (step=0023825) Train Loss: 3.7654, Train Steps/Sec: 1.10 +[2025-03-24 16:26:54] (step=0023850) Train Loss: 3.8013, Train Steps/Sec: 1.10 +[2025-03-24 16:27:18] (step=0023875) Train Loss: 3.7060, Train Steps/Sec: 1.06 +[2025-03-24 16:27:41] (step=0023900) Train Loss: 3.7552, Train Steps/Sec: 1.10 +[2025-03-24 16:28:04] (step=0023925) Train Loss: 3.8278, Train Steps/Sec: 1.07 +[2025-03-24 16:28:27] (step=0023950) Train Loss: 3.7485, Train Steps/Sec: 1.10 +[2025-03-24 16:28:50] (step=0023975) Train Loss: 3.7487, Train Steps/Sec: 1.05 +[2025-03-24 16:29:13] (step=0024000) Train Loss: 3.8438, Train Steps/Sec: 1.10 +[2025-03-24 16:29:13] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-03-24 16:38:16] Finish Eval in 24000 steps... +[2025-03-24 16:38:34] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0024000.pt +[2025-03-24 16:38:36] Removed old checkpoint: checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0020000.pt +[2025-03-24 16:38:59] (step=0024025) Train Loss: 3.8032, Train Steps/Sec: 0.04 +[2025-03-24 16:39:22] (step=0024050) Train Loss: 3.7325, Train Steps/Sec: 1.10 +[2025-03-24 16:39:45] (step=0024075) Train Loss: 3.7026, Train Steps/Sec: 1.09 +[2025-03-24 16:40:07] (step=0024100) Train Loss: 3.8385, Train Steps/Sec: 1.10 +[2025-03-24 16:40:30] (step=0024125) Train Loss: 3.6707, Train Steps/Sec: 1.09 +[2025-03-24 16:40:53] (step=0024150) Train Loss: 3.7718, Train Steps/Sec: 1.10 +[2025-03-24 16:41:16] (step=0024175) Train Loss: 3.8088, Train Steps/Sec: 1.10 +[2025-03-24 16:41:38] (step=0024200) Train Loss: 3.7400, Train Steps/Sec: 1.10 +[2025-03-24 16:42:01] (step=0024225) Train Loss: 3.7887, Train Steps/Sec: 1.10 +[2025-03-24 16:42:24] (step=0024250) Train Loss: 3.7431, Train Steps/Sec: 1.10 +[2025-03-24 16:42:47] (step=0024275) Train Loss: 3.7625, Train Steps/Sec: 1.10 +[2025-03-24 16:43:11] (step=0024300) Train Loss: 3.8486, Train Steps/Sec: 1.05 +[2025-03-24 16:43:35] (step=0024325) Train Loss: 3.7825, Train Steps/Sec: 1.01 +[2025-03-24 16:43:58] (step=0024350) Train Loss: 3.7757, Train Steps/Sec: 1.10 +[2025-03-24 16:44:22] (step=0024375) Train Loss: 3.8332, Train Steps/Sec: 1.05 +[2025-03-24 16:44:45] (step=0024400) Train Loss: 3.6699, Train Steps/Sec: 1.10 +[2025-03-24 16:45:07] (step=0024425) Train Loss: 3.7842, Train Steps/Sec: 1.10 +[2025-03-24 16:45:30] (step=0024450) Train Loss: 3.7548, Train Steps/Sec: 1.10 +[2025-03-24 16:45:53] (step=0024475) Train Loss: 3.8532, Train Steps/Sec: 1.10 +[2025-03-24 16:46:16] (step=0024500) Train Loss: 3.7482, Train Steps/Sec: 1.10 +[2025-03-24 16:46:39] (step=0024525) Train Loss: 3.7011, Train Steps/Sec: 1.10 +[2025-03-24 16:47:01] (step=0024550) Train Loss: 3.8169, Train Steps/Sec: 1.10 +[2025-03-24 16:47:26] (step=0024575) Train Loss: 3.7878, Train Steps/Sec: 1.01 +[2025-03-24 16:47:49] (step=0024600) Train Loss: 3.7843, Train Steps/Sec: 1.10 +[2025-03-24 16:48:12] (step=0024625) Train Loss: 3.8253, Train Steps/Sec: 1.10 +[2025-03-24 16:48:34] (step=0024650) Train Loss: 3.7400, Train Steps/Sec: 1.10 +[2025-03-24 16:48:57] (step=0024675) Train Loss: 3.8731, Train Steps/Sec: 1.10 +[2025-03-24 16:49:20] (step=0024700) Train Loss: 3.7555, Train Steps/Sec: 1.10 +[2025-03-24 16:49:43] (step=0024725) Train Loss: 3.8252, Train Steps/Sec: 1.10 +[2025-03-24 16:50:05] (step=0024750) Train Loss: 3.7180, Train Steps/Sec: 1.10 +[2025-03-24 16:50:28] (step=0024775) Train Loss: 3.7159, Train Steps/Sec: 1.10 +[2025-03-24 16:50:51] (step=0024800) Train Loss: 3.8233, Train Steps/Sec: 1.10 +[2025-03-24 16:51:15] (step=0024825) Train Loss: 3.8312, Train Steps/Sec: 1.05 +[2025-03-24 16:51:37] (step=0024850) Train Loss: 3.8430, Train Steps/Sec: 1.10 +[2025-03-24 16:52:00] (step=0024875) Train Loss: 3.7718, Train Steps/Sec: 1.10 +[2025-03-24 16:52:24] (step=0024900) Train Loss: 3.6848, Train Steps/Sec: 1.05 +[2025-03-24 16:52:48] (step=0024925) Train Loss: 3.8005, Train Steps/Sec: 1.05 +[2025-03-24 16:53:11] (step=0024950) Train Loss: 3.8165, Train Steps/Sec: 1.10 +[2025-03-24 16:53:33] (step=0024975) Train Loss: 3.7732, Train Steps/Sec: 1.10 +[2025-03-24 16:53:58] (step=0025000) Train Loss: 3.8033, Train Steps/Sec: 1.03 +[2025-03-24 16:54:20] (step=0025025) Train Loss: 3.6945, Train Steps/Sec: 1.10 +[2025-03-24 16:54:43] (step=0025050) Train Loss: 3.6173, Train Steps/Sec: 1.10 +[2025-03-24 16:55:06] (step=0025075) Train Loss: 3.7128, Train Steps/Sec: 1.10 +[2025-03-24 16:55:30] (step=0025100) Train Loss: 3.7611, Train Steps/Sec: 1.05 +[2025-03-24 16:55:53] (step=0025125) Train Loss: 3.7503, Train Steps/Sec: 1.09 +[2025-03-24 16:56:16] (step=0025150) Train Loss: 3.7286, Train Steps/Sec: 1.10 +[2025-03-24 16:56:39] (step=0025175) Train Loss: 3.6947, Train Steps/Sec: 1.10 +[2025-03-24 16:57:03] (step=0025200) Train Loss: 3.8427, Train Steps/Sec: 1.05 +[2025-03-24 16:57:26] (step=0025225) Train Loss: 3.8549, Train Steps/Sec: 1.10 +[2025-03-24 16:57:48] (step=0025250) Train Loss: 3.7157, Train Steps/Sec: 1.10 +[2025-03-24 16:58:11] (step=0025275) Train Loss: 3.8447, Train Steps/Sec: 1.10 +[2025-03-24 16:58:34] (step=0025300) Train Loss: 3.7399, Train Steps/Sec: 1.10 +[2025-03-24 16:58:57] (step=0025325) Train Loss: 3.7482, Train Steps/Sec: 1.10 +[2025-03-24 16:59:19] (step=0025350) Train Loss: 3.8129, Train Steps/Sec: 1.10 +[2025-03-24 16:59:42] (step=0025375) Train Loss: 3.7960, Train Steps/Sec: 1.10 +[2025-03-24 17:00:05] (step=0025400) Train Loss: 3.6053, Train Steps/Sec: 1.10 +[2025-03-24 17:00:28] (step=0025425) Train Loss: 3.8660, Train Steps/Sec: 1.10 +[2025-03-24 17:00:50] (step=0025450) Train Loss: 3.8269, Train Steps/Sec: 1.10 +[2025-03-24 17:01:13] (step=0025475) Train Loss: 3.7993, Train Steps/Sec: 1.10 +[2025-03-24 17:01:37] (step=0025500) Train Loss: 3.7457, Train Steps/Sec: 1.05 +[2025-03-24 17:02:00] (step=0025525) Train Loss: 3.7921, Train Steps/Sec: 1.10 +[2025-03-24 17:02:23] (step=0025550) Train Loss: 3.6621, Train Steps/Sec: 1.10 +[2025-03-24 17:02:47] (step=0025575) Train Loss: 3.7383, Train Steps/Sec: 1.02 +[2025-03-24 17:03:12] (step=0025600) Train Loss: 3.7648, Train Steps/Sec: 0.99 +[2025-03-24 17:03:35] (step=0025625) Train Loss: 3.7981, Train Steps/Sec: 1.10 +[2025-03-24 17:03:59] (step=0025650) Train Loss: 3.7416, Train Steps/Sec: 1.05 +[2025-03-24 17:04:22] (step=0025675) Train Loss: 3.7311, Train Steps/Sec: 1.07 +[2025-03-24 17:04:45] (step=0025700) Train Loss: 3.7593, Train Steps/Sec: 1.10 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/debug-internal.log b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f7dd89380c55af28d6ed794b69ba9a02c078c804 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2025-03-24T08:28:41.056200986Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug-core.log"} +{"time":"2025-03-24T08:28:41.189607189Z","level":"INFO","msg":"created new stream","id":"mlbv5b7s"} +{"time":"2025-03-24T08:28:41.189638487Z","level":"INFO","msg":"stream: started","id":"mlbv5b7s"} +{"time":"2025-03-24T08:28:41.189791641Z","level":"INFO","msg":"handler: started","stream_id":"mlbv5b7s"} +{"time":"2025-03-24T08:28:41.189819323Z","level":"INFO","msg":"writer: Do: started","stream_id":"mlbv5b7s"} +{"time":"2025-03-24T08:28:41.18982348Z","level":"INFO","msg":"sender: started","stream_id":"mlbv5b7s"} +{"time":"2025-03-24T08:28:41.484118777Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-24T17:02:13.70574757Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/debug.log b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b78bf9b6f3eed15fe248ca7684be0cfaa2bcb32b --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/debug.log @@ -0,0 +1,22 @@ +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_setup.py:_flush():67] Configure stats pid to 1395084 +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug.log +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug-internal.log +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_init.py:init():761] calling init triggers +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_dir/subject_ti2i_t2i_stage1_w_flux_segment_mid_700k.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoints', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoints/llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_newfix_train_only_projection_i2i/014-GPT-XL/checkpoints/0022000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 4, 'lr': 0.0005, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 4000, 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_dir/new_1117_validation_set.jsonl_mid_1k.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 512, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': False, 'no_replace': False, 'resume': False, 'dreambench_eval': False, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'i2i': False, 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-03-24 08:28:41,049 INFO MainThread:1395084 [wandb_init.py:init():784] starting backend +2025-03-24 08:28:41,049 INFO MainThread:1395084 [wandb_init.py:init():788] sending inform_init request +2025-03-24 08:28:41,054 INFO MainThread:1395084 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-03-24 08:28:41,054 INFO MainThread:1395084 [wandb_init.py:init():798] backend started and connected +2025-03-24 08:28:41,056 INFO MainThread:1395084 [wandb_init.py:init():891] updated telemetry +2025-03-24 08:28:41,087 INFO MainThread:1395084 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-03-24 08:28:41,481 INFO MainThread:1395084 [wandb_init.py:init():990] starting run threads in backend +2025-03-24 08:28:41,560 INFO MainThread:1395084 [wandb_run.py:_console_start():2375] atexit reg +2025-03-24 08:28:41,561 INFO MainThread:1395084 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-03-24 08:28:41,562 INFO MainThread:1395084 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-03-24 08:28:41,562 INFO MainThread:1395084 [wandb_run.py:_redirect():2315] Redirects installed. +2025-03-24 08:28:41,563 INFO MainThread:1395084 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/config.yaml b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aad132133ddf2b27d871fb78e5d3211aafbb2210 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/config.yaml @@ -0,0 +1,198 @@ +_wandb: + value: + cli_version: 0.19.8 + m: [] + python_version: 3.11.11 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "3": + - 13 + - 16 + - 23 + - 55 + "4": 3.11.11 + "5": 0.19.8 + "6": 4.49.0 + "8": + - 5 + "12": 0.19.8 + "13": linux-x86_64 +beta1: + value: 0.9 +beta2: + value: 0.95 +cfg_scale: + value: 7.5 +ckpt_every: + value: 4000 +class_dropout_prob: + value: 0.1 +cloud_save_path: + value: /tmp/haozhezhao/MLLMG/checkpoints +cls_token_num: + value: 512 +codebook_embed_dim: + value: 8 +codebook_size: + value: 16384 +continue_stage1: + value: false +data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_dir/subject_ti2i_t2i_stage1_w_flux_segment_mid_700k.jsonl +dataset: + value: ti2i +dist_backend: + value: nccl +dist_url: + value: env:// +distributed: + value: true +do_eval: + value: true +do_recovery: + value: false +downsample_size: + value: 16 +dreambench_eval: + value: false +drop_path: + value: 0 +dropout_p: + value: 0.1 +epochs: + value: 4 +eval_steps: + value: 2000 +find_unused_parameters: + value: true +global_batch_size: + value: 96 +global_seed: + value: 0 +gpt_ckpt: + value: /tmp/haozhezhao/MLLMG/checkpoints/llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_newfix_train_only_projection_i2i/014-GPT-XL/checkpoints/0022000.pt +gpt_model: + value: GPT-XL +gpt_type: + value: t2i +gpu: + value: 0 +gradient_accumulation_steps: + value: 1 +i2i: + value: false +image_only_rate: + value: 0.1 +image_place_holder: + value: +image_size: + value: 512 +load_fixed_llamagen: + value: true +load_from_checkpoint: + value: /tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt +load_language_projection: + value: /tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin +load_subject_embedding: + value: null +load_visual_encoder: + value: false +log_every: + value: 25 +lr: + value: 0.0005 +lr_decay_ratio: + value: 0.1 +lr_decay_style: + value: cosine +max_eval_samples: + value: 512 +max_grad_norm: + value: 1 +mixed_precision: + value: bf16 +mm_vision_tower: + value: openai/clip-vit-large-patch14 +model_name_or_path: + value: /tmp/haozhezhao/model/blip2-flan-t5-xl +multimodal_encoder: + value: llava +no_compile: + value: false +no_left_padding: + value: false +no_local_save: + value: false +no_replace: + value: false +num_classes: + value: 1000 +num_workers: + value: 4 +processor_path: + value: null +project_name: + value: llamagen_ti2i +rank: + value: 0 +reference_data_path: + value: /tmp/haozhezhao/MLLMG/cc12m_reference.jsonl +replace_subject: + value: false +results_dir: + value: checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all +resume: + value: false +save_total_limit: + value: 1 +stage2: + value: false +subject_driven: + value: true +temperature: + value: 0.9 +token_dropout_p: + value: 0.1 +top_k: + value: 16384 +top_p: + value: 1 +train_all: + value: true +train_iters: + value: 500000 +train_text_encoder: + value: true +unfreeze_output: + value: false +use_vision_tower: + value: true +val_data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_dir/new_1117_validation_set.jsonl_mid_1k.jsonl +vocab_size: + value: 16384 +vq_ckpt: + value: /tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt +vq_model: + value: VQ-16 +warmup: + value: 0.05 +weight_decay: + value: 0.05 +with_image_only: + value: false +world_size: + value: 8 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/output.log b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1f1231d15ff55a5dccf06f44b71ab0f120d88e71 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/output.log @@ -0,0 +1,264 @@ +[2025-03-24 08:00:43] Training for 4 epochs... +[2025-03-24 08:00:43] Beginning epoch 0... + 0%| | 0/25986 [00:00 + main(args) + File "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", line 635, in main + _, loss = model(cond_idx=c_indices, idx=z_indices[:,:-1],pixel_values=pixel_values,cond_idx_mask= cond_attn_mask,img_mask=image_masks, targets=z_indices, valid=valid, text_input_ids=text_input_ids, text_attention_mask=text_attention_mask) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward + else self._run_ddp_forward(*inputs, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward + return self.module(*inputs, **kwargs) # type: ignore[index] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 465, in _fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/MLLMG/autoregressive/models/gpt.py", line 460, in forward + def forward( + File "/tmp/haozhezhao/MLLMG/autoregressive/models/gpt.py", line 536, in torch_dynamo_resume_in_forward_at_477 + h = layer(h, freqs_cis, input_pos, mask, cls_token_num = rope_exclude_token_num ) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/MLLMG/autoregressive/models/gpt.py", line 286, in forward + def forward( + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 632, in _fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 784, in call_wrapped + return self._wrapped_call(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 361, in __call__ + raise e + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 348, in __call__ + return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File ".634", line 16, in forward + submod_1 = self.compiled_submod_1(submod_0, l_self_modules_attention_modules_wqkv_parameters_weight_, l_freqs_cis_, l_mask_, l_self_modules_attention_modules_wo_parameters_weight_, l_x_, l_self_modules_ffn_norm_parameters_weight_); submod_0 = l_self_modules_attention_modules_wqkv_parameters_weight_ = l_freqs_cis_ = l_mask_ = l_self_modules_attention_modules_wo_parameters_weight_ = l_x_ = l_self_modules_ffn_norm_parameters_weight_ = None + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/backends/distributed.py", line 154, in forward + x = self.submod(*args) + ^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 632, in _fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 1100, in forward + return compiled_fn(full_args) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 308, in runtime_wrapper + all_outs = call_func_at_runtime_with_args( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 124, in call_func_at_runtime_with_args + out = normalize_as_list(f(args)) + ^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 98, in g + return f(*args) + ^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/autograd/function.py", line 575, in apply + return super().apply(*args, **kwargs) # type: ignore[misc] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 1525, in forward + fw_outs = call_func_at_runtime_with_args( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 124, in call_func_at_runtime_with_args + out = normalize_as_list(f(args)) + ^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 579, in wrapper + return compiled_fn(runtime_args) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 488, in wrapper + return compiled_fn(runtime_args) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 667, in inner_fn + outs = compiled_fn(args) + ^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_inductor/codecache.py", line 1478, in __call__ + return self.current_callable(inputs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_inductor/utils.py", line 1977, in run + return model(new_inputs) + ^^^^^^^^^^^^^^^^^ + File "/tmp/torchinductor_haozhezhao/ag/cagajj4webmq4n7izf22o5zir4mz72mdj3gjy3lugcxw4fyufbfx.py", line 516, in call + buf7 = empty_strided_cuda((12, 20, 1, 1535, 64), (1964800, 98240, 98240, 64, 1), torch.bfloat16) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 46.00 MiB. GPU 0 has a total capacity of 79.15 GiB of which 33.69 MiB is free. Process 711814 has 79.11 GiB memory in use. Of the allocated memory 74.83 GiB is allocated by PyTorch, and 2.63 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank0]: Traceback (most recent call last): +[rank0]: File "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", line 1087, in +[rank0]: main(args) +[rank0]: File "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", line 635, in main +[rank0]: _, loss = model(cond_idx=c_indices, idx=z_indices[:,:-1],pixel_values=pixel_values,cond_idx_mask= cond_attn_mask,img_mask=image_masks, targets=z_indices, valid=valid, text_input_ids=text_input_ids, text_attention_mask=text_attention_mask) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward +[rank0]: else self._run_ddp_forward(*inputs, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward +[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index] +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 465, in _fn +[rank0]: return fn(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/MLLMG/autoregressive/models/gpt.py", line 460, in forward +[rank0]: def forward( +[rank0]: File "/tmp/haozhezhao/MLLMG/autoregressive/models/gpt.py", line 536, in torch_dynamo_resume_in_forward_at_477 +[rank0]: h = layer(h, freqs_cis, input_pos, mask, cls_token_num = rope_exclude_token_num ) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/MLLMG/autoregressive/models/gpt.py", line 286, in forward +[rank0]: def forward( +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 632, in _fn +[rank0]: return fn(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 784, in call_wrapped +[rank0]: return self._wrapped_call(self, *args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 361, in __call__ +[rank0]: raise e +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 348, in __call__ +[rank0]: return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File ".634", line 16, in forward +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/backends/distributed.py", line 154, in forward +[rank0]: x = self.submod(*args) +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 632, in _fn +[rank0]: return fn(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 1100, in forward +[rank0]: return compiled_fn(full_args) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 308, in runtime_wrapper +[rank0]: all_outs = call_func_at_runtime_with_args( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 124, in call_func_at_runtime_with_args +[rank0]: out = normalize_as_list(f(args)) +[rank0]: ^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 98, in g +[rank0]: return f(*args) +[rank0]: ^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/autograd/function.py", line 575, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 1525, in forward +[rank0]: fw_outs = call_func_at_runtime_with_args( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 124, in call_func_at_runtime_with_args +[rank0]: out = normalize_as_list(f(args)) +[rank0]: ^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 579, in wrapper +[rank0]: return compiled_fn(runtime_args) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 488, in wrapper +[rank0]: return compiled_fn(runtime_args) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 667, in inner_fn +[rank0]: outs = compiled_fn(args) +[rank0]: ^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_inductor/codecache.py", line 1478, in __call__ +[rank0]: return self.current_callable(inputs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_inductor/utils.py", line 1977, in run +[rank0]: return model(new_inputs) +[rank0]: ^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/torchinductor_haozhezhao/ag/cagajj4webmq4n7izf22o5zir4mz72mdj3gjy3lugcxw4fyufbfx.py", line 516, in call +[rank0]: buf7 = empty_strided_cuda((12, 20, 1, 1535, 64), (1964800, 98240, 98240, 64, 1), torch.bfloat16) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 46.00 MiB. GPU 0 has a total capacity of 79.15 GiB of which 33.69 MiB is free. Process 711814 has 79.11 GiB memory in use. Of the allocated memory 74.83 GiB is allocated by PyTorch, and 2.63 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/requirements.txt b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..d08a3e5e0c02054eefe2463e3487546131ecce79 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/requirements.txt @@ -0,0 +1,131 @@ +click==8.1.8 +networkx==3.3 +matplotlib-inline==0.1.7 +tqdm==4.67.1 +tzdata==2025.1 +pickleshare==0.7.5 +Jinja2==3.1.4 +opencv-python==4.11.0.86 +nvidia-cusparse-cu12==12.1.0.106 +fonttools==4.56.0 +ptyprocess==0.7.0 +nvidia-ml-py==12.570.86 +docker-pycreds==0.4.0 +datasets==3.4.0 +debugpy==1.8.13 +pillow==11.0.0 +charset-normalizer==3.4.1 +python-dateutil==2.9.0.post0 +wcwidth==0.2.13 +xxhash==3.5.0 +prompt_toolkit==3.0.50 +triton==3.1.0 +ipython==9.0.2 +aiosignal==1.3.2 +setuptools==75.8.0 +pyparsing==3.2.1 +tornado==6.4.2 +frozenlist==1.5.0 +requests==2.32.3 +parso==0.8.4 +MarkupSafe==2.1.5 +sentry-sdk==2.23.1 +comm==0.2.2 +contourpy==1.3.1 +certifi==2025.1.31 +propcache==0.3.0 +packaging==24.2 +idna==3.10 +nvitop==1.4.2 +pydantic_core==2.27.2 +pyarrow==19.0.1 +pexpect==4.9.0 +joblib==1.4.2 +fsspec==2024.12.0 +hf_transfer==0.1.9 +aiohttp==3.11.14 +wheel==0.45.1 +jupyter_client==8.6.3 +pytz==2025.1 +filelock==3.18.0 +multiprocess==0.70.16 +aiohappyeyeballs==2.6.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cublas-cu12==12.1.3.1 +importlib_metadata==8.6.1 +exceptiongroup==1.2.2 +urllib3==2.3.0 +tokenizers==0.21.1 +Pygments==2.19.1 +platformdirs==4.3.6 +mpmath==1.3.0 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-nvjitlink-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +regex==2024.11.6 +attrs==25.3.0 +asttokens==3.0.0 +pure_eval==0.2.3 +setproctitle==1.3.5 +kiwisolver==1.4.8 +nvidia-nccl-cu12==2.21.5 +torchaudio==2.5.1+cu121 +nvidia-cuda-nvrtc-cu12==12.1.105 +cycler==0.12.1 +PyYAML==6.0.2 +nest_asyncio==1.6.0 +jupyter_core==5.7.2 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +nvidia-cufft-cu12==11.0.2.54 +safetensors==0.5.3 +pip==25.0 +pyzmq==26.3.0 +smmap==5.0.2 +typing_extensions==4.12.2 +six==1.17.0 +torch==2.5.1+cu121 +zipp==3.21.0 +wandb==0.19.8 +gitdb==4.0.12 +decorator==5.2.1 +sympy==1.13.1 +traitlets==5.14.3 +numpy==2.2.4 +executing==2.1.0 +executing==2.2.0 +transformers==4.49.0 +protobuf==5.29.3 +annotated-types==0.7.0 +pandas==2.2.3 +jedi==0.19.2 +stack_data==0.6.3 +ipykernel==6.29.5 +psutil==7.0.0 +pydantic==2.10.6 +dill==0.3.8 +nltk==3.9.1 +multidict==6.1.0 +torchvision==0.20.1+cu121 +GitPython==3.1.44 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-nvtx-cu12==12.1.105 +nvidia-curand-cu12==10.3.2.106 +matplotlib==3.10.1 +huggingface-hub==0.29.3 +jaraco.text==3.12.1 +zipp==3.19.2 +jaraco.context==5.3.0 +autocommand==2.2.2 +inflect==7.3.1 +packaging==24.2 +jaraco.collections==5.1.0 +typeguard==4.3.0 +backports.tarfile==1.2.0 +tomli==2.0.1 +importlib_metadata==8.0.0 +typing_extensions==4.12.2 +jaraco.functools==4.0.1 +platformdirs==4.2.2 +wheel==0.43.0 +more-itertools==10.3.0 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/wandb-metadata.json b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e5398c022b6b71961bcab06819007459e5d325ae --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/wandb-metadata.json @@ -0,0 +1,149 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-03-24T08:00:42.238018Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_dir/subject_ti2i_t2i_stage1_w_flux_segment_mid_700k.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all", + "--cloud-save-path", + "/tmp/haozhezhao/MLLMG/checkpoints", + "--lr", + "5e-4", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_dir/new_1117_validation_set.jsonl_mid_1k.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/blip2-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "512", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "96", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "1", + "--train_text_encoder", + "--ckpt-every", + "4000", + "--epochs", + "4", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference.jsonl", + "--multimodal_encoder", + "llava", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--save_total_limit", + "1", + "--load_fixed_llamagen", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoints/llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_newfix_train_only_projection_i2i/014-GPT-XL/checkpoints/0022000.pt", + "--train_all" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "git": { + "remote": "git@github.com:HaozheZhao/MLLMG.git", + "commit": "f5d9894eac7b46fa35645a2cea9b04ee3b35f35b" + }, + "email": "mimazhe55360@gmail.com", + "root": "checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all", + "host": "447cc403a8794092814259713c51c1df00001D", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "73401270272" + } + }, + "memory": { + "total": "1902387908608" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/wandb-summary.json b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..88b216d6b683dea3652a852817f297efcf74ab65 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":96}} \ No newline at end of file diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug-core.log b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..66a704067d2202a275757a027f0c1ea442c35d19 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-24T08:00:41.484210456Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpwhblcku9/port-1386350.txt","pid":1386350,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-24T08:00:41.485861933Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1386350} +{"time":"2025-03-24T08:00:41.485839376Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":40861,"Zone":""}} +{"time":"2025-03-24T08:00:41.670833471Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:33506"} +{"time":"2025-03-24T08:00:42.239912165Z","level":"INFO","msg":"handleInformInit: received","streamId":"470gq1ph","id":"127.0.0.1:33506"} +{"time":"2025-03-24T08:00:42.448572105Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"470gq1ph","id":"127.0.0.1:33506"} +{"time":"2025-03-24T08:02:18.461071089Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:33506"} +{"time":"2025-03-24T08:02:18.461182761Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:33506"} +{"time":"2025-03-24T08:02:18.461317131Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:33506"} +{"time":"2025-03-24T08:02:18.461202788Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-24T08:02:19.582252545Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:33506"} +{"time":"2025-03-24T08:02:19.582291036Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:33506"} +{"time":"2025-03-24T08:02:19.58230952Z","level":"INFO","msg":"server is closed"} diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug-internal.log b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..37fb21812bac0bdd9bf1ad5d2b1e5dffded64fec --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-03-24T08:00:42.24021117Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug-core.log"} +{"time":"2025-03-24T08:00:42.448525408Z","level":"INFO","msg":"created new stream","id":"470gq1ph"} +{"time":"2025-03-24T08:00:42.448566534Z","level":"INFO","msg":"stream: started","id":"470gq1ph"} +{"time":"2025-03-24T08:00:42.448652949Z","level":"INFO","msg":"writer: Do: started","stream_id":"470gq1ph"} +{"time":"2025-03-24T08:00:42.44865848Z","level":"INFO","msg":"sender: started","stream_id":"470gq1ph"} +{"time":"2025-03-24T08:00:42.449017787Z","level":"INFO","msg":"handler: started","stream_id":"470gq1ph"} +{"time":"2025-03-24T08:00:42.987023753Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-24T08:02:18.461230245Z","level":"INFO","msg":"stream: closing","id":"470gq1ph"} +{"time":"2025-03-24T08:02:18.461271972Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-24T08:02:18.46219486Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-24T08:02:19.388178135Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-03-24T08:02:19.581945384Z","level":"INFO","msg":"handler: closed","stream_id":"470gq1ph"} +{"time":"2025-03-24T08:02:19.581974118Z","level":"INFO","msg":"writer: Close: closed","stream_id":"470gq1ph"} +{"time":"2025-03-24T08:02:19.582036032Z","level":"INFO","msg":"sender: closed","stream_id":"470gq1ph"} +{"time":"2025-03-24T08:02:19.582100372Z","level":"INFO","msg":"stream: closed","id":"470gq1ph"} diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug.log b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8476eb5c66182194440582c7d9e768ecca6aba20 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug.log @@ -0,0 +1,23 @@ +2025-03-24 08:00:42,232 INFO MainThread:1386350 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-03-24 08:00:42,232 INFO MainThread:1386350 [wandb_setup.py:_flush():67] Configure stats pid to 1386350 +2025-03-24 08:00:42,232 INFO MainThread:1386350 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-03-24 08:00:42,232 INFO MainThread:1386350 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-03-24 08:00:42,232 INFO MainThread:1386350 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-03-24 08:00:42,232 INFO MainThread:1386350 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug.log +2025-03-24 08:00:42,232 INFO MainThread:1386350 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/logs/debug-internal.log +2025-03-24 08:00:42,232 INFO MainThread:1386350 [wandb_init.py:init():761] calling init triggers +2025-03-24 08:00:42,232 INFO MainThread:1386350 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_dir/subject_ti2i_t2i_stage1_w_flux_segment_mid_700k.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoints', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoints/llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_newfix_train_only_projection_i2i/014-GPT-XL/checkpoints/0022000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 4, 'lr': 0.0005, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 96, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 4000, 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_dir/new_1117_validation_set.jsonl_mid_1k.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 512, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': False, 'no_replace': False, 'resume': False, 'dreambench_eval': False, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'i2i': False, 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-03-24 08:00:42,232 INFO MainThread:1386350 [wandb_init.py:init():784] starting backend +2025-03-24 08:00:42,232 INFO MainThread:1386350 [wandb_init.py:init():788] sending inform_init request +2025-03-24 08:00:42,237 INFO MainThread:1386350 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-03-24 08:00:42,237 INFO MainThread:1386350 [wandb_init.py:init():798] backend started and connected +2025-03-24 08:00:42,239 INFO MainThread:1386350 [wandb_init.py:init():891] updated telemetry +2025-03-24 08:00:42,245 INFO MainThread:1386350 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-03-24 08:00:42,984 INFO MainThread:1386350 [wandb_init.py:init():990] starting run threads in backend +2025-03-24 08:00:43,069 INFO MainThread:1386350 [wandb_run.py:_console_start():2375] atexit reg +2025-03-24 08:00:43,069 INFO MainThread:1386350 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-03-24 08:00:43,070 INFO MainThread:1386350 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-03-24 08:00:43,070 INFO MainThread:1386350 [wandb_run.py:_redirect():2315] Redirects installed. +2025-03-24 08:00:43,072 INFO MainThread:1386350 [wandb_init.py:init():1032] run started, returning control to user process +2025-03-24 08:02:18,460 INFO MsgRouterThr:1386350 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/run-470gq1ph.wandb b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/run-470gq1ph.wandb new file mode 100644 index 0000000000000000000000000000000000000000..115df41869629d92fe95ae468a1071c1d05b496f Binary files /dev/null and b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_080042-470gq1ph/run-470gq1ph.wandb differ diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/files/output.log b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a448dfa90fb4ac137197441fd66b4a5e3ead9620 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/files/output.log @@ -0,0 +1,1273 @@ +[2025-03-24 08:28:41] Training for 4 epochs... +[2025-03-24 08:28:41] Beginning epoch 0... + 0%| | 0/44547 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/9 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:56<00:00, 52.99s/it] +[2025-03-24 09:56:47] Finish Eval in 4000 steps...██████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:56<00:00, 52.83s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-03-24 09:57:05] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0004000.pt + 9%|███████████▌ | 4024/44547 [1:28:46<10:38:11, 1.06it/s][2025-03-24 09:57:29] (step=0004025) Train Loss: 3.9170, Train Steps/Sec: 0.04 + 9%|███████████▋ | 4049/44547 [1:29:09<10:12:08, 1.10it/s][2025-03-24 09:57:52] (step=0004050) Train Loss: 3.9599, Train Steps/Sec: 1.10 + 9%|███████████▋ | 4074/44547 [1:29:32<10:16:54, 1.09it/s][2025-03-24 09:58:14] (step=0004075) Train Loss: 4.1099, Train Steps/Sec: 1.10 + 9%|███████████▊ | 4099/44547 [1:29:55<10:21:11, 1.09it/s][2025-03-24 09:58:38] (step=0004100) Train Loss: 4.0088, Train Steps/Sec: 1.05 + 9%|███████████▊ | 4124/44547 [1:30:19<12:50:09, 1.14s/it][2025-03-24 09:59:02] (step=0004125) Train Loss: 3.9768, Train Steps/Sec: 1.06 + 9%|███████████▉ | 4149/44547 [1:30:43<10:12:29, 1.10it/s][2025-03-24 09:59:25] (step=0004150) Train Loss: 3.9963, Train Steps/Sec: 1.06 + 9%|███████████▉ | 4174/44547 [1:31:05<10:14:09, 1.10it/s][2025-03-24 09:59:48] (step=0004175) Train Loss: 3.9638, Train Steps/Sec: 1.10 + 9%|████████████ | 4199/44547 [1:31:28<10:11:12, 1.10it/s][2025-03-24 10:00:11] (step=0004200) Train Loss: 3.9720, Train Steps/Sec: 1.10 + 9%|████████████▏ | 4224/44547 [1:31:51<10:09:15, 1.10it/s][2025-03-24 10:00:33] (step=0004225) Train Loss: 3.9144, Train Steps/Sec: 1.10 + 10%|████████████▏ | 4249/44547 [1:32:13<10:10:01, 1.10it/s][2025-03-24 10:00:56] (step=0004250) Train Loss: 3.9741, Train Steps/Sec: 1.10 + 10%|████████████▎ | 4274/44547 [1:32:36<10:13:48, 1.09it/s][2025-03-24 10:01:19] (step=0004275) Train Loss: 3.9080, Train Steps/Sec: 1.10 + 10%|████████████▎ | 4299/44547 [1:32:59<10:12:14, 1.10it/s][2025-03-24 10:01:42] (step=0004300) Train Loss: 3.9658, Train Steps/Sec: 1.10 + 10%|████████████▍ | 4324/44547 [1:33:24<10:19:55, 1.08it/s][2025-03-24 10:02:06] (step=0004325) Train Loss: 3.9719, Train Steps/Sec: 1.01 + 10%|████████████▍ | 4349/44547 [1:33:46<10:08:53, 1.10it/s][2025-03-24 10:02:29] (step=0004350) Train Loss: 3.9709, Train Steps/Sec: 1.10 + 10%|████████████▌ | 4374/44547 [1:34:09<10:14:05, 1.09it/s][2025-03-24 10:02:52] (step=0004375) Train Loss: 3.9957, Train Steps/Sec: 1.10 + 10%|████████████▋ | 4399/44547 [1:34:32<10:10:11, 1.10it/s][2025-03-24 10:03:15] (step=0004400) Train Loss: 3.7936, Train Steps/Sec: 1.10 + 10%|████████████▋ | 4424/44547 [1:34:56<10:08:15, 1.10it/s][2025-03-24 10:03:39] (step=0004425) Train Loss: 4.1203, Train Steps/Sec: 1.05 + 10%|████████████▊ | 4449/44547 [1:35:19<10:06:36, 1.10it/s][2025-03-24 10:04:01] (step=0004450) Train Loss: 3.9797, Train Steps/Sec: 1.10 + 10%|████████████▊ | 4474/44547 [1:35:41<10:07:22, 1.10it/s][2025-03-24 10:04:24] (step=0004475) Train Loss: 4.0027, Train Steps/Sec: 1.10 + 10%|████████████▉ | 4499/44547 [1:36:04<10:12:30, 1.09it/s][2025-03-24 10:04:47] (step=0004500) Train Loss: 3.9635, Train Steps/Sec: 1.10 + 10%|████████████▉ | 4524/44547 [1:36:28<10:13:09, 1.09it/s][2025-03-24 10:05:11] (step=0004525) Train Loss: 4.0646, Train Steps/Sec: 1.06 + 10%|█████████████ | 4549/44547 [1:36:51<10:04:38, 1.10it/s][2025-03-24 10:05:33] (step=0004550) Train Loss: 3.9452, Train Steps/Sec: 1.10 + 10%|█████████████▏ | 4574/44547 [1:37:14<10:19:51, 1.07it/s][2025-03-24 10:05:57] (step=0004575) Train Loss: 4.0051, Train Steps/Sec: 1.05 + 10%|█████████████▏ | 4599/44547 [1:37:37<10:06:10, 1.10it/s][2025-03-24 10:06:20] (step=0004600) Train Loss: 4.0788, Train Steps/Sec: 1.10 + 10%|█████████████▎ | 4624/44547 [1:38:00<10:07:09, 1.10it/s][2025-03-24 10:06:43] (step=0004625) Train Loss: 3.9570, Train Steps/Sec: 1.10 + 10%|█████████████▎ | 4649/44547 [1:38:23<10:05:22, 1.10it/s][2025-03-24 10:07:05] (step=0004650) Train Loss: 3.9705, Train Steps/Sec: 1.10 + 10%|█████████████▍ | 4674/44547 [1:38:47<10:07:09, 1.09it/s][2025-03-24 10:07:29] (step=0004675) Train Loss: 4.0453, Train Steps/Sec: 1.05 + 11%|█████████████▌ | 4699/44547 [1:39:09<10:15:55, 1.08it/s][2025-03-24 10:07:52] (step=0004700) Train Loss: 4.0697, Train Steps/Sec: 1.10 + 11%|█████████████▌ | 4724/44547 [1:39:32<10:04:29, 1.10it/s][2025-03-24 10:08:15] (step=0004725) Train Loss: 4.0329, Train Steps/Sec: 1.10 + 11%|█████████████▋ | 4749/44547 [1:39:55<10:01:29, 1.10it/s][2025-03-24 10:08:38] (step=0004750) Train Loss: 3.8824, Train Steps/Sec: 1.10 + 11%|█████████████▋ | 4774/44547 [1:40:19<10:02:01, 1.10it/s][2025-03-24 10:09:01] (step=0004775) Train Loss: 3.9376, Train Steps/Sec: 1.06 + 11%|█████████████▊ | 4799/44547 [1:40:41<10:03:01, 1.10it/s][2025-03-24 10:09:24] (step=0004800) Train Loss: 3.9607, Train Steps/Sec: 1.10 + 11%|█████████████▊ | 4824/44547 [1:41:06<10:09:07, 1.09it/s][2025-03-24 10:09:48] (step=0004825) Train Loss: 3.9331, Train Steps/Sec: 1.03 + 11%|█████████████▉ | 4849/44547 [1:41:28<10:02:15, 1.10it/s][2025-03-24 10:10:11] (step=0004850) Train Loss: 4.0184, Train Steps/Sec: 1.10 + 11%|██████████████ | 4874/44547 [1:41:52<10:02:55, 1.10it/s][2025-03-24 10:10:35] (step=0004875) Train Loss: 4.0057, Train Steps/Sec: 1.04 + 11%|██████████████ | 4899/44547 [1:42:15<10:00:54, 1.10it/s][2025-03-24 10:10:58] (step=0004900) Train Loss: 4.0005, Train Steps/Sec: 1.10 + 11%|██████████████▏ | 4924/44547 [1:42:38<10:01:14, 1.10it/s][2025-03-24 10:11:21] (step=0004925) Train Loss: 3.8655, Train Steps/Sec: 1.10 + 11%|██████████████ | 4929/44547 [1:43:10<101:20:59, 9.21s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:632: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 11%|██████████████▏ | 4949/44547 [1:43:28<10:09:15, 1.08it/s][2025-03-24 10:12:11] (step=0004950) Train Loss: 4.0869, Train Steps/Sec: 0.50 + 11%|██████████████▎ | 4974/44547 [1:43:51<10:00:48, 1.10it/s][2025-03-24 10:12:34] (step=0004975) Train Loss: 3.9027, Train Steps/Sec: 1.10 + 11%|██████████████▎ | 4999/44547 [1:44:15<10:09:25, 1.08it/s][2025-03-24 10:12:58] (step=0005000) Train Loss: 3.9956, Train Steps/Sec: 1.05 + 11%|██████████████▍ | 5024/44547 [1:44:38<10:02:54, 1.09it/s][2025-03-24 10:13:21] (step=0005025) Train Loss: 3.9875, Train Steps/Sec: 1.09 + 11%|██████████████▌ | 5049/44547 [1:45:01<10:04:38, 1.09it/s][2025-03-24 10:13:43] (step=0005050) Train Loss: 3.9579, Train Steps/Sec: 1.09 + 11%|██████████████▌ | 5074/44547 [1:45:24<10:03:05, 1.09it/s][2025-03-24 10:14:06] (step=0005075) Train Loss: 3.8947, Train Steps/Sec: 1.10 + 11%|██████████████▊ | 5099/44547 [1:45:46<9:57:55, 1.10it/s][2025-03-24 10:14:29] (step=0005100) Train Loss: 3.9652, Train Steps/Sec: 1.10 + 12%|██████████████▊ | 5124/44547 [1:46:09<9:57:39, 1.10it/s][2025-03-24 10:14:52] (step=0005125) Train Loss: 4.0423, Train Steps/Sec: 1.10 + 12%|██████████████▊ | 5149/44547 [1:46:32<10:00:54, 1.09it/s][2025-03-24 10:15:15] (step=0005150) Train Loss: 4.0538, Train Steps/Sec: 1.09 + 12%|██████████████▉ | 5174/44547 [1:46:55<9:58:06, 1.10it/s][2025-03-24 10:15:37] (step=0005175) Train Loss: 4.0805, Train Steps/Sec: 1.10 + 12%|██████████████▉ | 5199/44547 [1:47:19<10:03:03, 1.09it/s][2025-03-24 10:16:01] (step=0005200) Train Loss: 4.0380, Train Steps/Sec: 1.05 + 12%|███████████████▏ | 5224/44547 [1:47:43<9:55:51, 1.10it/s][2025-03-24 10:16:26] (step=0005225) Train Loss: 3.9603, Train Steps/Sec: 1.02 + 12%|███████████████ | 5249/44547 [1:48:07<10:05:22, 1.08it/s][2025-03-24 10:16:49] (step=0005250) Train Loss: 3.9766, Train Steps/Sec: 1.05 + 12%|███████████████▎ | 5274/44547 [1:48:30<9:57:49, 1.09it/s][2025-03-24 10:17:12] (step=0005275) Train Loss: 3.9327, Train Steps/Sec: 1.09 + 12%|███████████████▎ | 5299/44547 [1:48:52<9:58:50, 1.09it/s][2025-03-24 10:17:35] (step=0005300) Train Loss: 4.0204, Train Steps/Sec: 1.10 + 12%|███████████████▍ | 5324/44547 [1:49:16<9:59:54, 1.09it/s][2025-03-24 10:17:59] (step=0005325) Train Loss: 4.0465, Train Steps/Sec: 1.06 + 12%|███████████████▍ | 5349/44547 [1:49:39<9:55:28, 1.10it/s][2025-03-24 10:18:21] (step=0005350) Train Loss: 3.9612, Train Steps/Sec: 1.10 + 12%|███████████████▌ | 5374/44547 [1:50:02<9:58:38, 1.09it/s][2025-03-24 10:18:44] (step=0005375) Train Loss: 4.0689, Train Steps/Sec: 1.09 + 12%|███████████████▋ | 5399/44547 [1:50:24<9:56:16, 1.09it/s][2025-03-24 10:19:07] (step=0005400) Train Loss: 3.9945, Train Steps/Sec: 1.09 + 12%|███████████████▋ | 5424/44547 [1:50:47<9:53:14, 1.10it/s][2025-03-24 10:19:30] (step=0005425) Train Loss: 3.9695, Train Steps/Sec: 1.10 + 12%|███████████████▊ | 5449/44547 [1:51:10<9:53:45, 1.10it/s][2025-03-24 10:19:53] (step=0005450) Train Loss: 3.9227, Train Steps/Sec: 1.09 + 12%|███████████████▊ | 5474/44547 [1:51:33<9:53:27, 1.10it/s][2025-03-24 10:20:16] (step=0005475) Train Loss: 3.9511, Train Steps/Sec: 1.10 + 12%|███████████████▉ | 5499/44547 [1:51:56<9:52:29, 1.10it/s][2025-03-24 10:20:38] (step=0005500) Train Loss: 4.0827, Train Steps/Sec: 1.10 + 12%|███████████████▉ | 5524/44547 [1:52:20<9:54:35, 1.09it/s][2025-03-24 10:21:02] (step=0005525) Train Loss: 4.0123, Train Steps/Sec: 1.04 + 12%|████████████████ | 5549/44547 [1:52:43<9:51:10, 1.10it/s][2025-03-24 10:21:25] (step=0005550) Train Loss: 4.1017, Train Steps/Sec: 1.10 + 13%|████████████████▏ | 5574/44547 [1:53:05<9:51:34, 1.10it/s][2025-03-24 10:21:48] (step=0005575) Train Loss: 4.0200, Train Steps/Sec: 1.10 + 13%|████████████████▏ | 5599/44547 [1:53:28<9:49:45, 1.10it/s][2025-03-24 10:22:11] (step=0005600) Train Loss: 4.0718, Train Steps/Sec: 1.10 + 13%|████████████████▎ | 5624/44547 [1:53:51<9:49:53, 1.10it/s][2025-03-24 10:22:34] (step=0005625) Train Loss: 4.0240, Train Steps/Sec: 1.10 + 13%|████████████████▎ | 5649/44547 [1:54:14<9:51:50, 1.10it/s][2025-03-24 10:22:56] (step=0005650) Train Loss: 4.0031, Train Steps/Sec: 1.10 + 13%|████████████████▍ | 5674/44547 [1:54:37<9:53:03, 1.09it/s][2025-03-24 10:23:19] (step=0005675) Train Loss: 3.9967, Train Steps/Sec: 1.10 + 13%|████████████████▍ | 5699/44547 [1:55:00<10:16:54, 1.05it/s][2025-03-24 10:23:43] (step=0005700) Train Loss: 4.0584, Train Steps/Sec: 1.05 + 13%|████████████████▍ | 5724/44547 [1:55:24<13:03:40, 1.21s/it][2025-03-24 10:24:07] (step=0005725) Train Loss: 4.0103, Train Steps/Sec: 1.05 + 13%|████████████████▋ | 5749/44547 [1:55:47<9:51:24, 1.09it/s][2025-03-24 10:24:30] (step=0005750) Train Loss: 3.9630, Train Steps/Sec: 1.10 + 13%|████████████████▋ | 5774/44547 [1:56:10<9:48:58, 1.10it/s][2025-03-24 10:24:52] (step=0005775) Train Loss: 4.1181, Train Steps/Sec: 1.09 + 13%|████████████████▊ | 5799/44547 [1:56:33<9:49:36, 1.10it/s][2025-03-24 10:25:15] (step=0005800) Train Loss: 3.9820, Train Steps/Sec: 1.09 + 13%|████████████████▊ | 5824/44547 [1:56:57<9:47:47, 1.10it/s][2025-03-24 10:25:40] (step=0005825) Train Loss: 4.0821, Train Steps/Sec: 1.01 + 13%|████████████████▉ | 5849/44547 [1:57:20<9:48:13, 1.10it/s][2025-03-24 10:26:03] (step=0005850) Train Loss: 3.9464, Train Steps/Sec: 1.10 + 13%|█████████████████ | 5874/44547 [1:57:43<9:46:11, 1.10it/s][2025-03-24 10:26:26] (step=0005875) Train Loss: 4.1394, Train Steps/Sec: 1.10 + 13%|█████████████████ | 5899/44547 [1:58:06<9:48:56, 1.09it/s][2025-03-24 10:26:48] (step=0005900) Train Loss: 3.9993, Train Steps/Sec: 1.10 + 13%|█████████████████▏ | 5924/44547 [1:58:29<9:46:40, 1.10it/s][2025-03-24 10:27:12] (step=0005925) Train Loss: 4.0358, Train Steps/Sec: 1.06 + 13%|█████████████████▏ | 5949/44547 [1:58:53<9:50:38, 1.09it/s][2025-03-24 10:27:36] (step=0005950) Train Loss: 3.9850, Train Steps/Sec: 1.05 + 13%|█████████████████▎ | 5974/44547 [1:59:16<9:48:21, 1.09it/s][2025-03-24 10:27:59] (step=0005975) Train Loss: 4.1222, Train Steps/Sec: 1.10 + 13%|█████████████████▎ | 5999/44547 [1:59:39<9:44:55, 1.10it/s][2025-03-24 10:28:21] (step=0006000) Train Loss: 4.1236, Train Steps/Sec: 1.10 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-03-24 10:28:21] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/9 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:54<00:00, 52.72s/it] +[2025-03-24 11:17:59] Finish Eval in 8000 steps...██████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:54<00:00, 52.56s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-03-24 11:18:20] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0008000.pt +[2025-03-24 11:18:22] Removed old checkpoint: checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0004000.pt + 18%|███████████████████████▏ | 8024/44547 [2:50:02<9:32:17, 1.06it/s][2025-03-24 11:18:45] (step=0008025) Train Loss: 4.0416, Train Steps/Sec: 0.04 + 18%|███████████████████████▎ | 8049/44547 [2:50:26<9:41:52, 1.05it/s][2025-03-24 11:19:09] (step=0008050) Train Loss: 4.1494, Train Steps/Sec: 1.05 + 18%|███████████████████████▍ | 8074/44547 [2:50:49<9:11:35, 1.10it/s][2025-03-24 11:19:32] (step=0008075) Train Loss: 4.0250, Train Steps/Sec: 1.10 + 18%|███████████████████████▍ | 8099/44547 [2:51:12<9:22:28, 1.08it/s][2025-03-24 11:19:55] (step=0008100) Train Loss: 3.9922, Train Steps/Sec: 1.07 + 18%|███████████████████████▌ | 8124/44547 [2:51:35<9:12:58, 1.10it/s][2025-03-24 11:20:18] (step=0008125) Train Loss: 3.9529, Train Steps/Sec: 1.10 + 18%|███████████████████████▌ | 8149/44547 [2:51:58<9:13:17, 1.10it/s][2025-03-24 11:20:40] (step=0008150) Train Loss: 4.0346, Train Steps/Sec: 1.10 + 18%|███████████████████████▋ | 8174/44547 [2:52:21<9:09:53, 1.10it/s][2025-03-24 11:21:03] (step=0008175) Train Loss: 4.0278, Train Steps/Sec: 1.10 + 18%|███████████████████████▋ | 8199/44547 [2:52:43<9:11:27, 1.10it/s][2025-03-24 11:21:26] (step=0008200) Train Loss: 4.0293, Train Steps/Sec: 1.10 + 18%|███████████████████████▊ | 8224/44547 [2:53:06<9:10:23, 1.10it/s][2025-03-24 11:21:49] (step=0008225) Train Loss: 4.0651, Train Steps/Sec: 1.10 + 19%|███████████████████████▉ | 8249/44547 [2:53:29<9:12:29, 1.09it/s][2025-03-24 11:22:11] (step=0008250) Train Loss: 4.1205, Train Steps/Sec: 1.10 + 19%|███████████████████████▉ | 8274/44547 [2:53:52<9:11:00, 1.10it/s][2025-03-24 11:22:34] (step=0008275) Train Loss: 3.9214, Train Steps/Sec: 1.10 + 19%|████████████████████████ | 8299/44547 [2:54:14<9:09:49, 1.10it/s][2025-03-24 11:22:57] (step=0008300) Train Loss: 4.2208, Train Steps/Sec: 1.10 + 19%|████████████████████████ | 8324/44547 [2:54:38<9:07:09, 1.10it/s][2025-03-24 11:23:20] (step=0008325) Train Loss: 4.1320, Train Steps/Sec: 1.07 + 19%|████████████████████████▏ | 8349/44547 [2:55:27<9:31:12, 1.06it/s][2025-03-24 11:24:10] (step=0008350) Train Loss: 3.9979, Train Steps/Sec: 0.51 + 19%|████████████████████████▏ | 8374/44547 [2:55:51<9:10:31, 1.10it/s][2025-03-24 11:24:33] (step=0008375) Train Loss: 4.0285, Train Steps/Sec: 1.05 + 19%|████████████████████████▎ | 8399/44547 [2:56:14<9:06:08, 1.10it/s][2025-03-24 11:24:56] (step=0008400) Train Loss: 4.1201, Train Steps/Sec: 1.10 + 19%|████████████████████████▍ | 8424/44547 [2:56:36<9:09:05, 1.10it/s][2025-03-24 11:25:19] (step=0008425) Train Loss: 4.0249, Train Steps/Sec: 1.10 + 19%|████████████████████████▍ | 8449/44547 [2:56:59<9:05:48, 1.10it/s][2025-03-24 11:25:42] (step=0008450) Train Loss: 3.9572, Train Steps/Sec: 1.10 + 19%|████████████████████████▌ | 8474/44547 [2:57:22<9:08:02, 1.10it/s][2025-03-24 11:26:06] (step=0008475) Train Loss: 4.0333, Train Steps/Sec: 1.05 + 19%|████████████████████████▌ | 8499/44547 [2:57:46<9:06:15, 1.10it/s][2025-03-24 11:26:28] (step=0008500) Train Loss: 4.0053, Train Steps/Sec: 1.10 + 19%|████████████████████████▋ | 8524/44547 [2:58:08<9:07:07, 1.10it/s][2025-03-24 11:26:51] (step=0008525) Train Loss: 4.0563, Train Steps/Sec: 1.10 + 19%|████████████████████████▊ | 8549/44547 [2:58:31<9:05:09, 1.10it/s][2025-03-24 11:27:14] (step=0008550) Train Loss: 4.0645, Train Steps/Sec: 1.10 + 19%|████████████████████████▊ | 8574/44547 [2:58:54<9:06:04, 1.10it/s][2025-03-24 11:27:37] (step=0008575) Train Loss: 4.0916, Train Steps/Sec: 1.10 + 19%|████████████████████████▋ | 8599/44547 [2:59:18<12:12:22, 1.22s/it][2025-03-24 11:28:00] (step=0008600) Train Loss: 4.1081, Train Steps/Sec: 1.05 + 19%|████████████████████████▉ | 8624/44547 [2:59:41<9:11:58, 1.08it/s][2025-03-24 11:28:23] (step=0008625) Train Loss: 3.9873, Train Steps/Sec: 1.10 + 19%|█████████████████████████ | 8649/44547 [3:00:03<9:02:35, 1.10it/s][2025-03-24 11:28:46] (step=0008650) Train Loss: 4.0245, Train Steps/Sec: 1.10 + 19%|█████████████████████████ | 8674/44547 [3:00:26<9:05:21, 1.10it/s][2025-03-24 11:29:09] (step=0008675) Train Loss: 3.9900, Train Steps/Sec: 1.10 + 20%|█████████████████████████▏ | 8699/44547 [3:00:50<9:11:25, 1.08it/s][2025-03-24 11:29:32] (step=0008700) Train Loss: 4.1167, Train Steps/Sec: 1.06 + 20%|█████████████████████████▎ | 8724/44547 [3:01:12<9:04:46, 1.10it/s][2025-03-24 11:29:55] (step=0008725) Train Loss: 4.1297, Train Steps/Sec: 1.10 + 20%|█████████████████████████▎ | 8749/44547 [3:01:35<9:03:25, 1.10it/s][2025-03-24 11:30:18] (step=0008750) Train Loss: 3.9935, Train Steps/Sec: 1.10 + 20%|█████████████████████████▍ | 8774/44547 [3:01:58<9:03:26, 1.10it/s][2025-03-24 11:30:41] (step=0008775) Train Loss: 4.0182, Train Steps/Sec: 1.10 + 20%|█████████████████████████▍ | 8799/44547 [3:02:21<9:03:59, 1.10it/s][2025-03-24 11:31:03] (step=0008800) Train Loss: 4.0240, Train Steps/Sec: 1.10 + 20%|█████████████████████████▌ | 8824/44547 [3:02:44<9:01:51, 1.10it/s][2025-03-24 11:31:26] (step=0008825) Train Loss: 4.0936, Train Steps/Sec: 1.10 + 20%|█████████████████████████▋ | 8849/44547 [3:03:07<9:00:35, 1.10it/s][2025-03-24 11:31:50] (step=0008850) Train Loss: 3.9998, Train Steps/Sec: 1.06 + 20%|█████████████████████████▋ | 8874/44547 [3:03:30<9:01:30, 1.10it/s][2025-03-24 11:32:12] (step=0008875) Train Loss: 4.0609, Train Steps/Sec: 1.10 + 20%|█████████████████████████▊ | 8899/44547 [3:03:54<9:20:28, 1.06it/s][2025-03-24 11:32:36] (step=0008900) Train Loss: 3.9739, Train Steps/Sec: 1.06 + 20%|█████████████████████████▊ | 8924/44547 [3:04:18<9:04:16, 1.09it/s][2025-03-24 11:33:00] (step=0008925) Train Loss: 3.9669, Train Steps/Sec: 1.04 + 20%|█████████████████████████▉ | 8949/44547 [3:04:40<9:00:49, 1.10it/s][2025-03-24 11:33:23] (step=0008950) Train Loss: 3.9677, Train Steps/Sec: 1.10 + 20%|█████████████████████████▉ | 8974/44547 [3:05:03<8:58:55, 1.10it/s][2025-03-24 11:33:46] (step=0008975) Train Loss: 3.9741, Train Steps/Sec: 1.10 + 20%|██████████████████████████ | 8999/44547 [3:05:26<8:58:54, 1.10it/s][2025-03-24 11:34:08] (step=0009000) Train Loss: 4.0596, Train Steps/Sec: 1.10 + 20%|██████████████████████████▏ | 9024/44547 [3:05:48<8:59:32, 1.10it/s][2025-03-24 11:34:31] (step=0009025) Train Loss: 4.0945, Train Steps/Sec: 1.10 + 20%|██████████████████████████ | 9049/44547 [3:06:12<10:02:44, 1.02s/it][2025-03-24 11:34:55] (step=0009050) Train Loss: 4.0836, Train Steps/Sec: 1.05 + 20%|██████████████████████████ | 9074/44547 [3:06:36<11:05:11, 1.13s/it][2025-03-24 11:35:19] (step=0009075) Train Loss: 4.0025, Train Steps/Sec: 1.05 + 20%|██████████████████████████▎ | 9099/44547 [3:06:59<8:56:12, 1.10it/s][2025-03-24 11:35:41] (step=0009100) Train Loss: 4.0727, Train Steps/Sec: 1.10 + 20%|██████████████████████████▍ | 9124/44547 [3:07:22<8:56:49, 1.10it/s][2025-03-24 11:36:04] (step=0009125) Train Loss: 4.0347, Train Steps/Sec: 1.10 + 21%|██████████████████████████▍ | 9149/44547 [3:07:45<8:57:05, 1.10it/s][2025-03-24 11:36:28] (step=0009150) Train Loss: 4.0263, Train Steps/Sec: 1.05 + 21%|██████████████████████████▌ | 9174/44547 [3:08:08<8:56:07, 1.10it/s][2025-03-24 11:36:51] (step=0009175) Train Loss: 4.0724, Train Steps/Sec: 1.10 + 21%|██████████████████████████▋ | 9199/44547 [3:08:32<8:58:16, 1.09it/s][2025-03-24 11:37:15] (step=0009200) Train Loss: 4.0422, Train Steps/Sec: 1.05 + 21%|██████████████████████████▋ | 9224/44547 [3:08:55<8:56:25, 1.10it/s][2025-03-24 11:37:37] (step=0009225) Train Loss: 4.0209, Train Steps/Sec: 1.10 + 21%|██████████████████████████▊ | 9249/44547 [3:09:18<8:55:30, 1.10it/s][2025-03-24 11:38:00] (step=0009250) Train Loss: 3.9129, Train Steps/Sec: 1.10 + 21%|██████████████████████████▊ | 9274/44547 [3:09:40<8:52:39, 1.10it/s][2025-03-24 11:38:23] (step=0009275) Train Loss: 4.0672, Train Steps/Sec: 1.10 + 21%|██████████████████████████▉ | 9299/44547 [3:10:03<8:55:40, 1.10it/s][2025-03-24 11:38:46] (step=0009300) Train Loss: 3.9437, Train Steps/Sec: 1.10 + 21%|███████████████████████████ | 9324/44547 [3:10:27<8:56:53, 1.09it/s][2025-03-24 11:39:09] (step=0009325) Train Loss: 3.9146, Train Steps/Sec: 1.06 + 21%|███████████████████████████ | 9349/44547 [3:10:49<8:55:08, 1.10it/s][2025-03-24 11:39:32] (step=0009350) Train Loss: 4.1373, Train Steps/Sec: 1.10 + 21%|███████████████████████████▏ | 9374/44547 [3:11:12<8:52:32, 1.10it/s][2025-03-24 11:39:55] (step=0009375) Train Loss: 3.9998, Train Steps/Sec: 1.10 + 21%|███████████████████████████▏ | 9399/44547 [3:11:35<8:54:02, 1.10it/s][2025-03-24 11:40:17] (step=0009400) Train Loss: 3.9705, Train Steps/Sec: 1.10 + 21%|███████████████████████████▎ | 9424/44547 [3:11:58<8:51:42, 1.10it/s][2025-03-24 11:40:40] (step=0009425) Train Loss: 3.8901, Train Steps/Sec: 1.09 + 21%|███████████████████████████▎ | 9449/44547 [3:12:20<8:50:11, 1.10it/s][2025-03-24 11:41:03] (step=0009450) Train Loss: 4.0317, Train Steps/Sec: 1.10 + 21%|███████████████████████████▍ | 9474/44547 [3:12:44<8:54:13, 1.09it/s][2025-03-24 11:41:27] (step=0009475) Train Loss: 3.9438, Train Steps/Sec: 1.06 + 21%|███████████████████████████▌ | 9499/44547 [3:13:07<8:51:31, 1.10it/s][2025-03-24 11:41:50] (step=0009500) Train Loss: 4.1032, Train Steps/Sec: 1.10 + 21%|███████████████████████████▌ | 9524/44547 [3:13:30<8:50:23, 1.10it/s][2025-03-24 11:42:12] (step=0009525) Train Loss: 4.0113, Train Steps/Sec: 1.10 + 21%|███████████████████████████▋ | 9549/44547 [3:13:52<8:50:37, 1.10it/s][2025-03-24 11:42:35] (step=0009550) Train Loss: 3.9810, Train Steps/Sec: 1.10 + 21%|███████████████████████████▌ | 9574/44547 [3:14:17<11:07:24, 1.15s/it][2025-03-24 11:43:00] (step=0009575) Train Loss: 3.9932, Train Steps/Sec: 1.01 + 22%|███████████████████████████▊ | 9599/44547 [3:14:40<8:51:15, 1.10it/s][2025-03-24 11:43:23] (step=0009600) Train Loss: 4.0821, Train Steps/Sec: 1.10 + 22%|███████████████████████████▊ | 9624/44547 [3:15:04<8:55:08, 1.09it/s][2025-03-24 11:43:46] (step=0009625) Train Loss: 4.1046, Train Steps/Sec: 1.05 + 22%|███████████████████████████▉ | 9649/44547 [3:15:26<8:49:26, 1.10it/s][2025-03-24 11:44:09] (step=0009650) Train Loss: 4.0197, Train Steps/Sec: 1.10 + 22%|████████████████████████████ | 9674/44547 [3:15:50<8:50:02, 1.10it/s][2025-03-24 11:44:33] (step=0009675) Train Loss: 4.1050, Train Steps/Sec: 1.05 + 22%|████████████████████████████ | 9699/44547 [3:16:13<8:47:29, 1.10it/s][2025-03-24 11:44:56] (step=0009700) Train Loss: 3.9146, Train Steps/Sec: 1.10 + 22%|████████████████████████████▏ | 9724/44547 [3:16:36<8:48:17, 1.10it/s][2025-03-24 11:45:18] (step=0009725) Train Loss: 3.9339, Train Steps/Sec: 1.10 + 22%|████████████████████████████▏ | 9749/44547 [3:16:59<8:49:36, 1.10it/s][2025-03-24 11:45:41] (step=0009750) Train Loss: 3.9330, Train Steps/Sec: 1.10 + 22%|████████████████████████████▎ | 9774/44547 [3:17:21<8:46:33, 1.10it/s][2025-03-24 11:46:04] (step=0009775) Train Loss: 4.1189, Train Steps/Sec: 1.10 + 22%|████████████████████████████▍ | 9799/44547 [3:17:46<9:21:50, 1.03it/s][2025-03-24 11:46:29] (step=0009800) Train Loss: 4.0059, Train Steps/Sec: 1.02 + 22%|████████████████████████████▍ | 9824/44547 [3:18:09<8:46:11, 1.10it/s][2025-03-24 11:46:51] (step=0009825) Train Loss: 3.9947, Train Steps/Sec: 1.10 + 22%|████████████████████████████▌ | 9849/44547 [3:18:32<8:45:49, 1.10it/s][2025-03-24 11:47:15] (step=0009850) Train Loss: 4.1284, Train Steps/Sec: 1.05 + 22%|████████████████████████████▌ | 9874/44547 [3:18:55<8:44:13, 1.10it/s][2025-03-24 11:47:38] (step=0009875) Train Loss: 4.1022, Train Steps/Sec: 1.10 + 22%|████████████████████████████▋ | 9899/44547 [3:19:18<8:45:52, 1.10it/s][2025-03-24 11:48:01] (step=0009900) Train Loss: 4.0145, Train Steps/Sec: 1.10 + 22%|████████████████████████████▋ | 9924/44547 [3:19:42<8:46:21, 1.10it/s][2025-03-24 11:48:24] (step=0009925) Train Loss: 4.1120, Train Steps/Sec: 1.05 + 22%|████████████████████████████▊ | 9949/44547 [3:20:04<8:47:02, 1.09it/s][2025-03-24 11:48:47] (step=0009950) Train Loss: 3.9351, Train Steps/Sec: 1.10 + 22%|████████████████████████████▉ | 9974/44547 [3:20:27<8:45:31, 1.10it/s][2025-03-24 11:49:10] (step=0009975) Train Loss: 3.9568, Train Steps/Sec: 1.10 + 22%|████████████████████████████▉ | 9999/44547 [3:20:50<8:47:35, 1.09it/s][2025-03-24 11:49:33] (step=0010000) Train Loss: 3.9528, Train Steps/Sec: 1.10 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-03-24 11:49:33] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:56<00:00, 52.89s/it] +[2025-03-24 11:58:36] Finish Eval in 10000 steps...█████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:55<00:00, 52.67s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... + 23%|████████████████████████████▊ | 10024/44547 [3:30:16<9:02:16, 1.06it/s][2025-03-24 11:58:59] (step=0010025) Train Loss: 3.9307, Train Steps/Sec: 0.04 + 23%|████████████████████████████▊ | 10049/44547 [3:30:39<8:47:01, 1.09it/s][2025-03-24 11:59:22] (step=0010050) Train Loss: 3.8784, Train Steps/Sec: 1.09 + 23%|████████████████████████████▉ | 10074/44547 [3:31:02<8:42:25, 1.10it/s][2025-03-24 11:59:45] (step=0010075) Train Loss: 4.0125, Train Steps/Sec: 1.10 + 23%|█████████████████████████████ | 10099/44547 [3:31:25<8:42:40, 1.10it/s][2025-03-24 12:00:07] (step=0010100) Train Loss: 4.0688, Train Steps/Sec: 1.10 + 23%|█████████████████████████████ | 10124/44547 [3:31:48<8:44:22, 1.09it/s][2025-03-24 12:00:31] (step=0010125) Train Loss: 4.0692, Train Steps/Sec: 1.06 + 23%|█████████████████████████████▏ | 10149/44547 [3:32:12<8:44:55, 1.09it/s][2025-03-24 12:00:55] (step=0010150) Train Loss: 4.0866, Train Steps/Sec: 1.05 + 23%|█████████████████████████████▏ | 10174/44547 [3:32:35<8:43:27, 1.09it/s][2025-03-24 12:01:18] (step=0010175) Train Loss: 3.9183, Train Steps/Sec: 1.10 + 23%|█████████████████████████████▎ | 10199/44547 [3:32:59<8:40:36, 1.10it/s][2025-03-24 12:01:41] (step=0010200) Train Loss: 3.9231, Train Steps/Sec: 1.05 + 23%|█████████████████████████████▍ | 10224/44547 [3:33:22<8:43:05, 1.09it/s][2025-03-24 12:02:04] (step=0010225) Train Loss: 4.0445, Train Steps/Sec: 1.10 + 23%|█████████████████████████████▍ | 10249/44547 [3:33:44<8:38:26, 1.10it/s][2025-03-24 12:02:27] (step=0010250) Train Loss: 4.0809, Train Steps/Sec: 1.10 + 23%|█████████████████████████████▌ | 10274/44547 [3:34:07<8:40:47, 1.10it/s][2025-03-24 12:02:50] (step=0010275) Train Loss: 4.0954, Train Steps/Sec: 1.10 + 23%|█████████████████████████████▌ | 10299/44547 [3:34:30<8:41:20, 1.09it/s][2025-03-24 12:03:13] (step=0010300) Train Loss: 3.9544, Train Steps/Sec: 1.10 + 23%|█████████████████████████████▋ | 10324/44547 [3:34:53<8:39:47, 1.10it/s][2025-03-24 12:03:35] (step=0010325) Train Loss: 4.0629, Train Steps/Sec: 1.10 + 23%|█████████████████████████████▋ | 10349/44547 [3:35:15<8:39:25, 1.10it/s][2025-03-24 12:03:58] (step=0010350) Train Loss: 3.9266, Train Steps/Sec: 1.10 + 23%|█████████████████████████████▊ | 10374/44547 [3:35:38<8:38:31, 1.10it/s][2025-03-24 12:04:21] (step=0010375) Train Loss: 3.9386, Train Steps/Sec: 1.10 + 23%|█████████████████████████████▉ | 10399/44547 [3:36:01<8:37:07, 1.10it/s][2025-03-24 12:04:44] (step=0010400) Train Loss: 3.9641, Train Steps/Sec: 1.10 + 23%|█████████████████████████████▉ | 10424/44547 [3:36:24<8:37:35, 1.10it/s][2025-03-24 12:05:06] (step=0010425) Train Loss: 3.9107, Train Steps/Sec: 1.10 + 23%|██████████████████████████████ | 10449/44547 [3:36:46<8:35:53, 1.10it/s][2025-03-24 12:05:30] (step=0010450) Train Loss: 3.9594, Train Steps/Sec: 1.10 + 24%|██████████████████████████████ | 10474/44547 [3:37:11<8:43:09, 1.09it/s][2025-03-24 12:05:54] (step=0010475) Train Loss: 4.0157, Train Steps/Sec: 1.07 + 24%|██████████████████████████████▏ | 10499/44547 [3:37:34<8:36:35, 1.10it/s][2025-03-24 12:06:17] (step=0010500) Train Loss: 4.0387, Train Steps/Sec: 1.10 + 24%|██████████████████████████████▏ | 10524/44547 [3:37:57<8:37:17, 1.10it/s][2025-03-24 12:06:39] (step=0010525) Train Loss: 4.0182, Train Steps/Sec: 1.10 + 24%|██████████████████████████████▎ | 10549/44547 [3:38:20<8:35:38, 1.10it/s][2025-03-24 12:07:03] (step=0010550) Train Loss: 4.1250, Train Steps/Sec: 1.05 + 24%|██████████████████████████████▍ | 10574/44547 [3:38:43<8:35:12, 1.10it/s][2025-03-24 12:07:26] (step=0010575) Train Loss: 3.9902, Train Steps/Sec: 1.10 + 24%|██████████████████████████████▍ | 10599/44547 [3:39:06<8:35:20, 1.10it/s][2025-03-24 12:07:49] (step=0010600) Train Loss: 3.8836, Train Steps/Sec: 1.10 + 24%|██████████████████████████████▌ | 10624/44547 [3:39:29<8:33:52, 1.10it/s][2025-03-24 12:08:11] (step=0010625) Train Loss: 3.9598, Train Steps/Sec: 1.10 + 24%|██████████████████████████████▌ | 10649/44547 [3:39:52<8:33:39, 1.10it/s][2025-03-24 12:08:34] (step=0010650) Train Loss: 3.9453, Train Steps/Sec: 1.10 + 24%|██████████████████████████████▋ | 10674/44547 [3:40:16<8:51:23, 1.06it/s][2025-03-24 12:08:59] (step=0010675) Train Loss: 3.9836, Train Steps/Sec: 1.00 + 24%|██████████████████████████████▋ | 10699/44547 [3:40:39<8:33:26, 1.10it/s][2025-03-24 12:09:22] (step=0010700) Train Loss: 3.8844, Train Steps/Sec: 1.10 + 24%|██████████████████████████████▊ | 10724/44547 [3:41:03<8:32:49, 1.10it/s][2025-03-24 12:09:45] (step=0010725) Train Loss: 4.0067, Train Steps/Sec: 1.06 + 24%|██████████████████████████████▉ | 10749/44547 [3:41:26<8:31:13, 1.10it/s][2025-03-24 12:10:08] (step=0010750) Train Loss: 3.9032, Train Steps/Sec: 1.10 + 24%|██████████████████████████████▉ | 10774/44547 [3:41:48<8:31:41, 1.10it/s][2025-03-24 12:10:31] (step=0010775) Train Loss: 3.9612, Train Steps/Sec: 1.10 + 24%|███████████████████████████████ | 10799/44547 [3:42:11<8:31:37, 1.10it/s][2025-03-24 12:10:54] (step=0010800) Train Loss: 3.8929, Train Steps/Sec: 1.10 + 24%|███████████████████████████████ | 10824/44547 [3:42:34<8:34:46, 1.09it/s][2025-03-24 12:11:17] (step=0010825) Train Loss: 3.9190, Train Steps/Sec: 1.10 + 24%|███████████████████████████████▏ | 10849/44547 [3:42:57<8:29:37, 1.10it/s][2025-03-24 12:11:39] (step=0010850) Train Loss: 4.1196, Train Steps/Sec: 1.10 + 24%|███████████████████████████████▏ | 10874/44547 [3:43:19<8:31:00, 1.10it/s][2025-03-24 12:12:02] (step=0010875) Train Loss: 3.8741, Train Steps/Sec: 1.10 + 24%|███████████████████████████████▎ | 10899/44547 [3:43:42<8:29:35, 1.10it/s][2025-03-24 12:12:25] (step=0010900) Train Loss: 3.9729, Train Steps/Sec: 1.10 + 25%|███████████████████████████████▍ | 10924/44547 [3:44:05<8:29:13, 1.10it/s][2025-03-24 12:12:48] (step=0010925) Train Loss: 4.0194, Train Steps/Sec: 1.10 + 25%|███████████████████████████████▍ | 10949/44547 [3:44:28<8:30:21, 1.10it/s][2025-03-24 12:13:10] (step=0010950) Train Loss: 3.8576, Train Steps/Sec: 1.10 + 25%|███████████████████████████████▌ | 10974/44547 [3:44:52<8:32:06, 1.09it/s][2025-03-24 12:13:34] (step=0010975) Train Loss: 3.9707, Train Steps/Sec: 1.05 + 25%|███████████████████████████████▌ | 10999/44547 [3:45:15<8:27:34, 1.10it/s][2025-03-24 12:13:58] (step=0011000) Train Loss: 4.0065, Train Steps/Sec: 1.05 + 25%|███████████████████████████████▋ | 11024/44547 [3:45:38<8:27:32, 1.10it/s][2025-03-24 12:14:21] (step=0011025) Train Loss: 4.0142, Train Steps/Sec: 1.10 + 25%|███████████████████████████████▋ | 11049/44547 [3:46:01<8:30:11, 1.09it/s][2025-03-24 12:14:44] (step=0011050) Train Loss: 3.9616, Train Steps/Sec: 1.10 + 25%|███████████████████████████████▊ | 11074/44547 [3:46:24<8:27:48, 1.10it/s][2025-03-24 12:15:06] (step=0011075) Train Loss: 3.8998, Train Steps/Sec: 1.10 + 25%|███████████████████████████████▉ | 11099/44547 [3:46:48<8:34:22, 1.08it/s][2025-03-24 12:15:30] (step=0011100) Train Loss: 3.9133, Train Steps/Sec: 1.05 + 25%|███████████████████████████████▉ | 11124/44547 [3:47:10<8:25:27, 1.10it/s][2025-03-24 12:15:53] (step=0011125) Train Loss: 3.9706, Train Steps/Sec: 1.10 + 25%|████████████████████████████████ | 11149/44547 [3:47:33<8:27:50, 1.10it/s][2025-03-24 12:16:16] (step=0011150) Train Loss: 4.0264, Train Steps/Sec: 1.07 + 25%|████████████████████████████████ | 11174/44547 [3:47:57<8:25:58, 1.10it/s][2025-03-24 12:16:39] (step=0011175) Train Loss: 3.8168, Train Steps/Sec: 1.10 + 25%|████████████████████████████████▏ | 11199/44547 [3:48:19<8:24:24, 1.10it/s][2025-03-24 12:17:02] (step=0011200) Train Loss: 4.0246, Train Steps/Sec: 1.10 + 25%|████████████████████████████████▎ | 11224/44547 [3:48:43<8:25:13, 1.10it/s][2025-03-24 12:17:26] (step=0011225) Train Loss: 3.9986, Train Steps/Sec: 1.05 + 25%|████████████████████████████████▎ | 11249/44547 [3:49:07<8:24:01, 1.10it/s][2025-03-24 12:17:49] (step=0011250) Train Loss: 4.0381, Train Steps/Sec: 1.06 + 25%|████████████████████████████████▍ | 11274/44547 [3:49:29<8:25:16, 1.10it/s][2025-03-24 12:18:12] (step=0011275) Train Loss: 3.9025, Train Steps/Sec: 1.10 + 25%|████████████████████████████████▍ | 11299/44547 [3:49:52<8:24:09, 1.10it/s][2025-03-24 12:18:35] (step=0011300) Train Loss: 3.8730, Train Steps/Sec: 1.10 + 25%|████████████████████████████████▌ | 11324/44547 [3:50:15<8:23:50, 1.10it/s][2025-03-24 12:18:58] (step=0011325) Train Loss: 4.0631, Train Steps/Sec: 1.10 + 25%|████████████████████████████████▌ | 11349/44547 [3:50:38<8:23:12, 1.10it/s][2025-03-24 12:19:21] (step=0011350) Train Loss: 3.9104, Train Steps/Sec: 1.06 + 26%|████████████████████████████████▋ | 11374/44547 [3:51:01<8:24:26, 1.10it/s][2025-03-24 12:19:44] (step=0011375) Train Loss: 3.9969, Train Steps/Sec: 1.10 + 26%|████████████████████████████████▊ | 11399/44547 [3:51:24<8:22:04, 1.10it/s][2025-03-24 12:20:07] (step=0011400) Train Loss: 3.8362, Train Steps/Sec: 1.10 + 26%|████████████████████████████████▊ | 11424/44547 [3:51:47<8:22:21, 1.10it/s][2025-03-24 12:20:29] (step=0011425) Train Loss: 3.9240, Train Steps/Sec: 1.10 + 26%|████████████████████████████████▉ | 11449/44547 [3:52:09<8:22:11, 1.10it/s][2025-03-24 12:20:52] (step=0011450) Train Loss: 3.9425, Train Steps/Sec: 1.10 + 26%|████████████████████████████████▉ | 11474/44547 [3:52:33<8:22:00, 1.10it/s][2025-03-24 12:21:16] (step=0011475) Train Loss: 3.9817, Train Steps/Sec: 1.05 + 26%|█████████████████████████████████ | 11499/44547 [3:52:56<8:23:02, 1.09it/s][2025-03-24 12:21:39] (step=0011500) Train Loss: 4.0620, Train Steps/Sec: 1.10 + 26%|█████████████████████████████████ | 11524/44547 [3:53:19<8:19:27, 1.10it/s][2025-03-24 12:22:01] (step=0011525) Train Loss: 3.9479, Train Steps/Sec: 1.10 + 26%|█████████████████████████████████▏ | 11549/44547 [3:53:42<8:19:51, 1.10it/s][2025-03-24 12:22:24] (step=0011550) Train Loss: 3.9652, Train Steps/Sec: 1.10 + 26%|█████████████████████████████████▎ | 11574/44547 [3:54:04<8:20:08, 1.10it/s][2025-03-24 12:22:47] (step=0011575) Train Loss: 3.9575, Train Steps/Sec: 1.10 + 26%|█████████████████████████████████▎ | 11599/44547 [3:54:28<8:30:26, 1.08it/s][2025-03-24 12:23:11] (step=0011600) Train Loss: 3.9184, Train Steps/Sec: 1.06 + 26%|█████████████████████████████████▍ | 11624/44547 [3:54:51<8:22:54, 1.09it/s][2025-03-24 12:23:33] (step=0011625) Train Loss: 3.9976, Train Steps/Sec: 1.09 + 26%|█████████████████████████████████▍ | 11649/44547 [3:55:14<8:19:11, 1.10it/s][2025-03-24 12:23:56] (step=0011650) Train Loss: 4.0163, Train Steps/Sec: 1.10 + 26%|█████████████████████████████████▌ | 11674/44547 [3:55:36<8:20:21, 1.09it/s][2025-03-24 12:24:19] (step=0011675) Train Loss: 3.9339, Train Steps/Sec: 1.10 + 26%|█████████████████████████████████▌ | 11699/44547 [3:56:00<8:17:20, 1.10it/s][2025-03-24 12:24:43] (step=0011700) Train Loss: 4.0896, Train Steps/Sec: 1.05 + 26%|█████████████████████████████████▋ | 11724/44547 [3:56:24<8:21:31, 1.09it/s][2025-03-24 12:25:07] (step=0011725) Train Loss: 3.9238, Train Steps/Sec: 1.05 + 26%|█████████████████████████████████▊ | 11749/44547 [3:56:48<8:17:05, 1.10it/s][2025-03-24 12:25:30] (step=0011750) Train Loss: 3.8668, Train Steps/Sec: 1.05 + 26%|█████████████████████████████████▊ | 11774/44547 [3:57:11<8:17:31, 1.10it/s][2025-03-24 12:25:53] (step=0011775) Train Loss: 3.9434, Train Steps/Sec: 1.10 + 26%|█████████████████████████████████▉ | 11799/44547 [3:57:34<8:17:41, 1.10it/s][2025-03-24 12:26:17] (step=0011800) Train Loss: 3.9786, Train Steps/Sec: 1.05 + 27%|█████████████████████████████████▉ | 11824/44547 [3:57:58<8:16:33, 1.10it/s][2025-03-24 12:26:41] (step=0011825) Train Loss: 3.9668, Train Steps/Sec: 1.05 + 27%|██████████████████████████████████ | 11849/44547 [3:58:21<8:19:02, 1.09it/s][2025-03-24 12:27:04] (step=0011850) Train Loss: 3.9481, Train Steps/Sec: 1.10 + 27%|██████████████████████████████████ | 11874/44547 [3:58:44<8:16:33, 1.10it/s][2025-03-24 12:27:26] (step=0011875) Train Loss: 3.9912, Train Steps/Sec: 1.10 + 27%|██████████████████████████████████▏ | 11899/44547 [3:59:06<8:15:04, 1.10it/s][2025-03-24 12:27:49] (step=0011900) Train Loss: 3.8915, Train Steps/Sec: 1.10 + 27%|██████████████████████████████████▎ | 11924/44547 [3:59:29<8:14:00, 1.10it/s][2025-03-24 12:28:12] (step=0011925) Train Loss: 3.9910, Train Steps/Sec: 1.10 + 27%|██████████████████████████████████▎ | 11949/44547 [3:59:52<8:14:58, 1.10it/s][2025-03-24 12:28:35] (step=0011950) Train Loss: 3.9491, Train Steps/Sec: 1.09 + 27%|██████████████████████████████████▍ | 11974/44547 [4:00:16<8:45:04, 1.03it/s][2025-03-24 12:28:58] (step=0011975) Train Loss: 3.8943, Train Steps/Sec: 1.06 + 27%|██████████████████████████████████▍ | 11999/44547 [4:00:38<8:14:03, 1.10it/s][2025-03-24 12:29:21] (step=0012000) Train Loss: 3.9327, Train Steps/Sec: 1.10 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-03-24 12:29:21] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:54<00:00, 52.75s/it] +[2025-03-24 12:38:23] Finish Eval in 12000 steps...█████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:54<00:00, 52.61s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-03-24 12:38:41] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0012000.pt +[2025-03-24 12:38:43] Removed old checkpoint: checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0008000.pt + 27%|██████████████████████████████████▌ | 12024/44547 [4:10:24<8:31:25, 1.06it/s][2025-03-24 12:39:06] (step=0012025) Train Loss: 3.9945, Train Steps/Sec: 0.04 + 27%|██████████████████████████████████▌ | 12049/44547 [4:10:46<8:13:05, 1.10it/s][2025-03-24 12:39:29] (step=0012050) Train Loss: 3.9142, Train Steps/Sec: 1.10 + 27%|██████████████████████████████████▋ | 12074/44547 [4:11:09<8:11:43, 1.10it/s][2025-03-24 12:39:52] (step=0012075) Train Loss: 4.0554, Train Steps/Sec: 1.10 + 27%|██████████████████████████████████▊ | 12099/44547 [4:11:32<8:12:23, 1.10it/s][2025-03-24 12:40:14] (step=0012100) Train Loss: 3.9243, Train Steps/Sec: 1.10 + 27%|██████████████████████████████████▊ | 12124/44547 [4:11:55<8:12:12, 1.10it/s][2025-03-24 12:40:37] (step=0012125) Train Loss: 3.9639, Train Steps/Sec: 1.10 + 27%|██████████████████████████████████▉ | 12149/44547 [4:12:17<8:13:26, 1.09it/s][2025-03-24 12:41:00] (step=0012150) Train Loss: 3.8675, Train Steps/Sec: 1.10 + 27%|██████████████████████████████████▉ | 12174/44547 [4:12:40<8:12:54, 1.09it/s][2025-03-24 12:41:23] (step=0012175) Train Loss: 3.8129, Train Steps/Sec: 1.10 + 27%|███████████████████████████████████ | 12199/44547 [4:13:03<8:09:56, 1.10it/s][2025-03-24 12:41:46] (step=0012200) Train Loss: 3.8501, Train Steps/Sec: 1.10 + 27%|███████████████████████████████████ | 12224/44547 [4:13:26<8:10:46, 1.10it/s][2025-03-24 12:42:08] (step=0012225) Train Loss: 3.9917, Train Steps/Sec: 1.10 + 27%|███████████████████████████████████▏ | 12249/44547 [4:13:48<8:10:05, 1.10it/s][2025-03-24 12:42:31] (step=0012250) Train Loss: 3.8778, Train Steps/Sec: 1.10 + 28%|███████████████████████████████████▎ | 12274/44547 [4:14:12<8:08:43, 1.10it/s][2025-03-24 12:42:55] (step=0012275) Train Loss: 3.9570, Train Steps/Sec: 1.05 + 28%|███████████████████████████████████▎ | 12299/44547 [4:14:36<8:08:55, 1.10it/s][2025-03-24 12:43:18] (step=0012300) Train Loss: 3.9565, Train Steps/Sec: 1.06 + 28%|███████████████████████████████████▍ | 12324/44547 [4:14:59<8:11:02, 1.09it/s][2025-03-24 12:43:41] (step=0012325) Train Loss: 3.8473, Train Steps/Sec: 1.10 + 28%|███████████████████████████████████▍ | 12349/44547 [4:15:21<8:08:04, 1.10it/s][2025-03-24 12:44:04] (step=0012350) Train Loss: 4.0417, Train Steps/Sec: 1.10 + 28%|███████████████████████████████████▌ | 12374/44547 [4:15:44<8:09:06, 1.10it/s][2025-03-24 12:44:27] (step=0012375) Train Loss: 4.0909, Train Steps/Sec: 1.10 + 28%|███████████████████████████████████▋ | 12399/44547 [4:16:09<8:27:58, 1.05it/s][2025-03-24 12:44:51] (step=0012400) Train Loss: 3.9523, Train Steps/Sec: 1.01 + 28%|███████████████████████████████████▋ | 12424/44547 [4:16:32<8:07:09, 1.10it/s][2025-03-24 12:45:14] (step=0012425) Train Loss: 3.9533, Train Steps/Sec: 1.10 + 28%|███████████████████████████████████▊ | 12449/44547 [4:16:54<8:07:27, 1.10it/s][2025-03-24 12:45:37] (step=0012450) Train Loss: 3.9483, Train Steps/Sec: 1.10 + 28%|███████████████████████████████████▊ | 12474/44547 [4:17:17<8:17:54, 1.07it/s][2025-03-24 12:46:00] (step=0012475) Train Loss: 4.0402, Train Steps/Sec: 1.09 + 28%|███████████████████████████████████▉ | 12499/44547 [4:17:40<8:05:54, 1.10it/s][2025-03-24 12:46:23] (step=0012500) Train Loss: 4.0832, Train Steps/Sec: 1.10 + 28%|███████████████████████████████████▉ | 12524/44547 [4:18:04<8:05:56, 1.10it/s][2025-03-24 12:46:46] (step=0012525) Train Loss: 3.9001, Train Steps/Sec: 1.05 + 28%|████████████████████████████████████ | 12549/44547 [4:18:27<8:04:01, 1.10it/s][2025-03-24 12:47:09] (step=0012550) Train Loss: 3.8924, Train Steps/Sec: 1.10 + 28%|████████████████████████████████████▏ | 12574/44547 [4:18:50<8:10:04, 1.09it/s][2025-03-24 12:47:33] (step=0012575) Train Loss: 4.0073, Train Steps/Sec: 1.05 + 28%|████████████████████████████████████▏ | 12599/44547 [4:19:13<8:05:53, 1.10it/s][2025-03-24 12:47:56] (step=0012600) Train Loss: 3.9703, Train Steps/Sec: 1.10 + 28%|████████████████████████████████████▎ | 12624/44547 [4:19:37<8:12:59, 1.08it/s][2025-03-24 12:48:19] (step=0012625) Train Loss: 3.8759, Train Steps/Sec: 1.06 + 28%|████████████████████████████████████▎ | 12649/44547 [4:19:59<8:04:06, 1.10it/s][2025-03-24 12:48:42] (step=0012650) Train Loss: 3.9104, Train Steps/Sec: 1.10 + 28%|████████████████████████████████████▍ | 12674/44547 [4:20:22<8:04:20, 1.10it/s][2025-03-24 12:49:05] (step=0012675) Train Loss: 3.9331, Train Steps/Sec: 1.10 + 29%|████████████████████████████████████▍ | 12699/44547 [4:20:46<9:06:40, 1.03s/it][2025-03-24 12:49:29] (step=0012700) Train Loss: 3.9353, Train Steps/Sec: 1.04 + 29%|████████████████████████████████████▌ | 12724/44547 [4:21:09<8:03:56, 1.10it/s][2025-03-24 12:49:52] (step=0012725) Train Loss: 3.9271, Train Steps/Sec: 1.10 + 29%|████████████████████████████████████▋ | 12749/44547 [4:21:32<8:01:03, 1.10it/s][2025-03-24 12:50:14] (step=0012750) Train Loss: 3.9157, Train Steps/Sec: 1.10 + 29%|████████████████████████████████████▋ | 12774/44547 [4:21:55<8:03:50, 1.09it/s][2025-03-24 12:50:37] (step=0012775) Train Loss: 3.9823, Train Steps/Sec: 1.09 + 29%|████████████████████████████████████▊ | 12799/44547 [4:22:17<8:05:31, 1.09it/s][2025-03-24 12:51:00] (step=0012800) Train Loss: 3.9138, Train Steps/Sec: 1.09 + 29%|████████████████████████████████████▊ | 12824/44547 [4:22:40<8:01:45, 1.10it/s][2025-03-24 12:51:23] (step=0012825) Train Loss: 4.0440, Train Steps/Sec: 1.10 + 29%|████████████████████████████████████▉ | 12849/44547 [4:23:03<8:05:16, 1.09it/s][2025-03-24 12:51:46] (step=0012850) Train Loss: 4.0150, Train Steps/Sec: 1.10 + 29%|████████████████████████████████████▉ | 12874/44547 [4:23:26<7:57:51, 1.10it/s][2025-03-24 12:52:08] (step=0012875) Train Loss: 3.8485, Train Steps/Sec: 1.10 + 29%|█████████████████████████████████████ | 12899/44547 [4:23:49<8:37:27, 1.02it/s][2025-03-24 12:52:32] (step=0012900) Train Loss: 4.0198, Train Steps/Sec: 1.06 + 29%|█████████████████████████████████████▏ | 12924/44547 [4:24:12<7:58:25, 1.10it/s][2025-03-24 12:52:55] (step=0012925) Train Loss: 3.9467, Train Steps/Sec: 1.10 + 29%|█████████████████████████████████████▏ | 12949/44547 [4:24:35<7:58:07, 1.10it/s][2025-03-24 12:53:17] (step=0012950) Train Loss: 3.9934, Train Steps/Sec: 1.10 + 29%|█████████████████████████████████████▎ | 12974/44547 [4:24:58<8:16:23, 1.06it/s][2025-03-24 12:53:41] (step=0012975) Train Loss: 3.9119, Train Steps/Sec: 1.07 + 29%|█████████████████████████████████████▎ | 12999/44547 [4:25:22<7:56:06, 1.10it/s][2025-03-24 12:54:05] (step=0013000) Train Loss: 3.8157, Train Steps/Sec: 1.05 + 29%|█████████████████████████████████████▍ | 13024/44547 [4:25:46<8:57:22, 1.02s/it][2025-03-24 12:54:29] (step=0013025) Train Loss: 3.9825, Train Steps/Sec: 1.05 + 29%|█████████████████████████████████████▍ | 13049/44547 [4:26:09<8:00:07, 1.09it/s][2025-03-24 12:54:51] (step=0013050) Train Loss: 3.9612, Train Steps/Sec: 1.10 + 29%|█████████████████████████████████████▌ | 13074/44547 [4:26:31<7:56:24, 1.10it/s][2025-03-24 12:55:14] (step=0013075) Train Loss: 3.9100, Train Steps/Sec: 1.10 + 29%|█████████████████████████████████████▋ | 13099/44547 [4:26:54<7:54:51, 1.10it/s][2025-03-24 12:55:37] (step=0013100) Train Loss: 3.9186, Train Steps/Sec: 1.10 + 29%|█████████████████████████████████████▋ | 13124/44547 [4:27:17<7:56:01, 1.10it/s][2025-03-24 12:55:59] (step=0013125) Train Loss: 3.9599, Train Steps/Sec: 1.10 + 30%|█████████████████████████████████████▊ | 13149/44547 [4:27:40<8:00:58, 1.09it/s][2025-03-24 12:56:22] (step=0013150) Train Loss: 3.8642, Train Steps/Sec: 1.10 + 30%|█████████████████████████████████████▊ | 13174/44547 [4:28:02<7:54:40, 1.10it/s][2025-03-24 12:56:45] (step=0013175) Train Loss: 3.9165, Train Steps/Sec: 1.10 + 30%|█████████████████████████████████████▉ | 13199/44547 [4:28:25<7:54:26, 1.10it/s][2025-03-24 12:57:08] (step=0013200) Train Loss: 3.9382, Train Steps/Sec: 1.10 + 30%|█████████████████████████████████████▉ | 13224/44547 [4:28:48<7:56:22, 1.10it/s][2025-03-24 12:57:30] (step=0013225) Train Loss: 3.9720, Train Steps/Sec: 1.10 + 30%|██████████████████████████████████████ | 13249/44547 [4:29:12<8:06:11, 1.07it/s][2025-03-24 12:57:54] (step=0013250) Train Loss: 3.8577, Train Steps/Sec: 1.05 + 30%|██████████████████████████████████████▏ | 13274/44547 [4:29:34<7:52:42, 1.10it/s][2025-03-24 12:58:17] (step=0013275) Train Loss: 3.9132, Train Steps/Sec: 1.10 + 30%|██████████████████████████████████████▏ | 13299/44547 [4:29:57<7:52:03, 1.10it/s][2025-03-24 12:58:40] (step=0013300) Train Loss: 3.9652, Train Steps/Sec: 1.10 + 30%|██████████████████████████████████████▎ | 13324/44547 [4:30:20<7:52:22, 1.10it/s][2025-03-24 12:59:02] (step=0013325) Train Loss: 4.0022, Train Steps/Sec: 1.10 + 30%|██████████████████████████████████████▎ | 13349/44547 [4:30:42<7:52:26, 1.10it/s][2025-03-24 12:59:25] (step=0013350) Train Loss: 4.0367, Train Steps/Sec: 1.10 + 30%|██████████████████████████████████████▍ | 13374/44547 [4:31:05<7:51:50, 1.10it/s][2025-03-24 12:59:48] (step=0013375) Train Loss: 3.9371, Train Steps/Sec: 1.10 + 30%|██████████████████████████████████████▌ | 13399/44547 [4:31:30<8:43:26, 1.01s/it][2025-03-24 13:00:13] (step=0013400) Train Loss: 3.8963, Train Steps/Sec: 1.00 + 30%|██████████████████████████████████████▌ | 13424/44547 [4:31:54<7:50:08, 1.10it/s][2025-03-24 13:00:37] (step=0013425) Train Loss: 3.9196, Train Steps/Sec: 1.05 + 30%|██████████████████████████████████████▋ | 13449/44547 [4:32:17<7:51:00, 1.10it/s][2025-03-24 13:01:00] (step=0013450) Train Loss: 3.9423, Train Steps/Sec: 1.07 + 30%|██████████████████████████████████████▋ | 13474/44547 [4:32:40<7:53:00, 1.09it/s][2025-03-24 13:01:23] (step=0013475) Train Loss: 3.9701, Train Steps/Sec: 1.10 + 30%|██████████████████████████████████████▊ | 13499/44547 [4:33:03<7:53:07, 1.09it/s][2025-03-24 13:01:45] (step=0013500) Train Loss: 3.9220, Train Steps/Sec: 1.10 + 30%|██████████████████████████████████████▊ | 13524/44547 [4:33:26<7:50:50, 1.10it/s][2025-03-24 13:02:08] (step=0013525) Train Loss: 3.9357, Train Steps/Sec: 1.10 + 30%|██████████████████████████████████████▉ | 13549/44547 [4:33:48<7:47:31, 1.11it/s][2025-03-24 13:02:31] (step=0013550) Train Loss: 3.8928, Train Steps/Sec: 1.10 + 30%|███████████████████████████████████████ | 13574/44547 [4:34:11<7:47:14, 1.10it/s][2025-03-24 13:02:54] (step=0013575) Train Loss: 3.9551, Train Steps/Sec: 1.10 + 31%|███████████████████████████████████████ | 13599/44547 [4:34:35<8:07:32, 1.06it/s][2025-03-24 13:03:17] (step=0013600) Train Loss: 3.9482, Train Steps/Sec: 1.06 + 31%|███████████████████████████████████████▏ | 13624/44547 [4:34:58<7:46:03, 1.11it/s][2025-03-24 13:03:41] (step=0013625) Train Loss: 3.8540, Train Steps/Sec: 1.05 + 31%|███████████████████████████████████████▏ | 13649/44547 [4:35:21<7:46:48, 1.10it/s][2025-03-24 13:04:04] (step=0013650) Train Loss: 3.9671, Train Steps/Sec: 1.10 + 31%|███████████████████████████████████████▎ | 13674/44547 [4:35:44<7:46:38, 1.10it/s][2025-03-24 13:04:26] (step=0013675) Train Loss: 3.9184, Train Steps/Sec: 1.10 + 31%|███████████████████████████████████████▎ | 13699/44547 [4:36:08<7:52:58, 1.09it/s][2025-03-24 13:04:50] (step=0013700) Train Loss: 3.9963, Train Steps/Sec: 1.05 + 31%|███████████████████████████████████████▍ | 13724/44547 [4:36:30<7:45:52, 1.10it/s][2025-03-24 13:05:13] (step=0013725) Train Loss: 3.9830, Train Steps/Sec: 1.10 + 31%|███████████████████████████████████████▌ | 13749/44547 [4:36:53<7:45:08, 1.10it/s][2025-03-24 13:05:36] (step=0013750) Train Loss: 3.9521, Train Steps/Sec: 1.10 + 31%|███████████████████████████████████████▌ | 13774/44547 [4:37:26<9:16:27, 1.08s/it][2025-03-24 13:06:08] (step=0013775) Train Loss: 3.9510, Train Steps/Sec: 0.76 + 31%|███████████████████████████████████████▋ | 13799/44547 [4:37:48<7:44:36, 1.10it/s][2025-03-24 13:06:31] (step=0013800) Train Loss: 3.8771, Train Steps/Sec: 1.10 + 31%|███████████████████████████████████████▋ | 13824/44547 [4:38:11<7:45:06, 1.10it/s][2025-03-24 13:06:54] (step=0013825) Train Loss: 3.9367, Train Steps/Sec: 1.10 + 31%|███████████████████████████████████████▊ | 13849/44547 [4:38:34<7:43:28, 1.10it/s][2025-03-24 13:07:17] (step=0013850) Train Loss: 3.8606, Train Steps/Sec: 1.10 + 31%|███████████████████████████████████████▊ | 13874/44547 [4:38:57<7:44:18, 1.10it/s][2025-03-24 13:07:39] (step=0013875) Train Loss: 3.9089, Train Steps/Sec: 1.10 + 31%|███████████████████████████████████████▉ | 13899/44547 [4:39:19<7:44:38, 1.10it/s][2025-03-24 13:08:02] (step=0013900) Train Loss: 3.8423, Train Steps/Sec: 1.10 + 31%|████████████████████████████████████████ | 13924/44547 [4:39:42<7:41:46, 1.11it/s][2025-03-24 13:08:25] (step=0013925) Train Loss: 3.9317, Train Steps/Sec: 1.10 + 31%|████████████████████████████████████████ | 13949/44547 [4:40:05<7:44:14, 1.10it/s][2025-03-24 13:08:47] (step=0013950) Train Loss: 3.8049, Train Steps/Sec: 1.10 + 31%|████████████████████████████████████████▏ | 13974/44547 [4:40:28<8:17:29, 1.02it/s][2025-03-24 13:09:11] (step=0013975) Train Loss: 3.8479, Train Steps/Sec: 1.06 + 31%|████████████████████████████████████████▏ | 13999/44547 [4:40:51<7:41:02, 1.10it/s][2025-03-24 13:09:34] (step=0014000) Train Loss: 3.9517, Train Steps/Sec: 1.10 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-03-24 13:09:34] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:59<00:00, 53.25s/it] +[2025-03-24 13:18:41] Finish Eval in 14000 steps...███████████████████████████████████████████████████████| 9/9 [07:58<00:00, 53.04s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... + 31%|████████████████████████████████████████▎ | 14024/44547 [4:50:22<7:59:37, 1.06it/s][2025-03-24 13:19:05] (step=0014025) Train Loss: 3.8639, Train Steps/Sec: 0.04 + 32%|████████████████████████████████████████▎ | 14049/44547 [4:50:45<7:40:55, 1.10it/s][2025-03-24 13:19:27] (step=0014050) Train Loss: 3.8946, Train Steps/Sec: 1.10 + 32%|████████████████████████████████████████▍ | 14074/44547 [4:51:08<7:41:22, 1.10it/s][2025-03-24 13:19:50] (step=0014075) Train Loss: 3.8724, Train Steps/Sec: 1.10 + 32%|████████████████████████████████████████▌ | 14099/44547 [4:51:30<7:41:00, 1.10it/s][2025-03-24 13:20:13] (step=0014100) Train Loss: 3.8866, Train Steps/Sec: 1.10 + 32%|████████████████████████████████████████▌ | 14124/44547 [4:51:54<7:41:25, 1.10it/s][2025-03-24 13:20:36] (step=0014125) Train Loss: 3.9471, Train Steps/Sec: 1.07 + 32%|████████████████████████████████████████▋ | 14149/44547 [4:52:17<7:40:49, 1.10it/s][2025-03-24 13:21:00] (step=0014150) Train Loss: 3.8986, Train Steps/Sec: 1.06 + 32%|████████████████████████████████████████▋ | 14174/44547 [4:52:41<7:39:42, 1.10it/s][2025-03-24 13:21:24] (step=0014175) Train Loss: 4.0240, Train Steps/Sec: 1.05 + 32%|████████████████████████████████████████▊ | 14199/44547 [4:53:05<7:53:27, 1.07it/s][2025-03-24 13:21:47] (step=0014200) Train Loss: 3.8876, Train Steps/Sec: 1.06 + 32%|████████████████████████████████████████▊ | 14224/44547 [4:53:27<7:41:26, 1.10it/s][2025-03-24 13:22:10] (step=0014225) Train Loss: 3.8100, Train Steps/Sec: 1.10 + 32%|████████████████████████████████████████▉ | 14249/44547 [4:53:50<7:37:51, 1.10it/s][2025-03-24 13:22:33] (step=0014250) Train Loss: 3.9576, Train Steps/Sec: 1.10 + 32%|█████████████████████████████████████████ | 14274/44547 [4:54:13<7:38:59, 1.10it/s][2025-03-24 13:22:56] (step=0014275) Train Loss: 3.8309, Train Steps/Sec: 1.10 + 32%|█████████████████████████████████████████ | 14299/44547 [4:54:36<7:36:39, 1.10it/s][2025-03-24 13:23:18] (step=0014300) Train Loss: 3.9339, Train Steps/Sec: 1.10 + 32%|█████████████████████████████████████████▏ | 14324/44547 [4:54:58<7:36:59, 1.10it/s][2025-03-24 13:23:41] (step=0014325) Train Loss: 3.8710, Train Steps/Sec: 1.10 + 32%|█████████████████████████████████████████▏ | 14349/44547 [4:55:22<8:01:31, 1.05it/s][2025-03-24 13:24:05] (step=0014350) Train Loss: 4.0023, Train Steps/Sec: 1.04 + 32%|█████████████████████████████████████████▎ | 14374/44547 [4:55:46<7:37:10, 1.10it/s][2025-03-24 13:24:29] (step=0014375) Train Loss: 4.0265, Train Steps/Sec: 1.05 + 32%|█████████████████████████████████████████▎ | 14399/44547 [4:56:09<7:40:12, 1.09it/s][2025-03-24 13:24:51] (step=0014400) Train Loss: 3.7750, Train Steps/Sec: 1.10 + 32%|█████████████████████████████████████████▍ | 14424/44547 [4:56:31<7:36:23, 1.10it/s][2025-03-24 13:25:14] (step=0014425) Train Loss: 4.0044, Train Steps/Sec: 1.10 + 32%|█████████████████████████████████████████▌ | 14449/44547 [4:56:54<7:38:40, 1.09it/s][2025-03-24 13:25:37] (step=0014450) Train Loss: 3.7259, Train Steps/Sec: 1.10 + 32%|█████████████████████████████████████████▌ | 14474/44547 [4:57:17<7:35:19, 1.10it/s][2025-03-24 13:26:00] (step=0014475) Train Loss: 3.8937, Train Steps/Sec: 1.10 + 33%|█████████████████████████████████████████▋ | 14499/44547 [4:57:40<7:34:52, 1.10it/s][2025-03-24 13:26:22] (step=0014500) Train Loss: 3.7247, Train Steps/Sec: 1.10 + 33%|█████████████████████████████████████████▋ | 14524/44547 [4:58:02<7:37:21, 1.09it/s][2025-03-24 13:26:45] (step=0014525) Train Loss: 3.8539, Train Steps/Sec: 1.10 + 33%|█████████████████████████████████████████▊ | 14549/44547 [4:58:25<7:33:11, 1.10it/s][2025-03-24 13:27:08] (step=0014550) Train Loss: 3.9759, Train Steps/Sec: 1.10 + 33%|█████████████████████████████████████████▉ | 14574/44547 [4:58:48<7:32:07, 1.10it/s][2025-03-24 13:27:30] (step=0014575) Train Loss: 3.8687, Train Steps/Sec: 1.10 + 33%|█████████████████████████████████████████▉ | 14599/44547 [4:59:10<7:32:27, 1.10it/s][2025-03-24 13:27:53] (step=0014600) Train Loss: 3.8824, Train Steps/Sec: 1.10 + 33%|██████████████████████████████████████████ | 14624/44547 [4:59:33<7:31:17, 1.11it/s][2025-03-24 13:28:16] (step=0014625) Train Loss: 3.9429, Train Steps/Sec: 1.10 + 33%|██████████████████████████████████████████ | 14649/44547 [4:59:56<7:34:24, 1.10it/s][2025-03-24 13:28:38] (step=0014650) Train Loss: 3.9459, Train Steps/Sec: 1.10 + 33%|██████████████████████████████████████████▏ | 14674/44547 [5:00:19<7:31:06, 1.10it/s][2025-03-24 13:29:01] (step=0014675) Train Loss: 3.9577, Train Steps/Sec: 1.10 + 33%|██████████████████████████████████████████▏ | 14699/44547 [5:00:41<7:31:22, 1.10it/s][2025-03-24 13:29:24] (step=0014700) Train Loss: 3.8390, Train Steps/Sec: 1.10 + 33%|██████████████████████████████████████████▎ | 14724/44547 [5:01:04<7:30:40, 1.10it/s][2025-03-24 13:29:46] (step=0014725) Train Loss: 3.7382, Train Steps/Sec: 1.10 + 33%|██████████████████████████████████████████▍ | 14749/44547 [5:01:28<7:30:14, 1.10it/s][2025-03-24 13:30:10] (step=0014750) Train Loss: 3.9281, Train Steps/Sec: 1.06 + 33%|██████████████████████████████████████████▍ | 14774/44547 [5:01:52<7:50:50, 1.05it/s][2025-03-24 13:30:35] (step=0014775) Train Loss: 3.9619, Train Steps/Sec: 1.01 + 33%|██████████████████████████████████████████▌ | 14799/44547 [5:02:16<7:35:49, 1.09it/s][2025-03-24 13:30:58] (step=0014800) Train Loss: 3.8351, Train Steps/Sec: 1.07 + 33%|██████████████████████████████████████████▌ | 14824/44547 [5:02:38<7:29:34, 1.10it/s][2025-03-24 13:31:21] (step=0014825) Train Loss: 3.8463, Train Steps/Sec: 1.10 + 33%|██████████████████████████████████████████▋ | 14849/44547 [5:03:01<7:28:21, 1.10it/s][2025-03-24 13:31:44] (step=0014850) Train Loss: 3.8982, Train Steps/Sec: 1.10 + 33%|██████████████████████████████████████████▋ | 14874/44547 [5:03:25<8:51:00, 1.07s/it][2025-03-24 13:32:08] (step=0014875) Train Loss: 3.9652, Train Steps/Sec: 1.05 + 33%|██████████████████████████████████████████▊ | 14899/44547 [5:03:48<7:27:50, 1.10it/s][2025-03-24 13:32:30] (step=0014900) Train Loss: 3.9260, Train Steps/Sec: 1.10 + 34%|██████████████████████████████████████████▉ | 14924/44547 [5:04:11<7:27:44, 1.10it/s][2025-03-24 13:32:54] (step=0014925) Train Loss: 4.0408, Train Steps/Sec: 1.06 + 34%|██████████████████████████████████████████▉ | 14949/44547 [5:04:34<7:30:00, 1.10it/s][2025-03-24 13:33:17] (step=0014950) Train Loss: 3.8804, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████ | 14974/44547 [5:04:57<7:26:16, 1.10it/s][2025-03-24 13:33:39] (step=0014975) Train Loss: 3.8686, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████ | 14999/44547 [5:05:19<7:27:24, 1.10it/s][2025-03-24 13:34:02] (step=0015000) Train Loss: 3.9142, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████▏ | 15024/44547 [5:05:42<7:25:28, 1.10it/s][2025-03-24 13:34:25] (step=0015025) Train Loss: 4.0365, Train Steps/Sec: 1.11 + 34%|███████████████████████████████████████████▏ | 15049/44547 [5:06:07<7:34:18, 1.08it/s][2025-03-24 13:34:49] (step=0015050) Train Loss: 3.7829, Train Steps/Sec: 1.01 + 34%|███████████████████████████████████████████▎ | 15074/44547 [5:06:29<7:25:32, 1.10it/s][2025-03-24 13:35:12] (step=0015075) Train Loss: 3.9863, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████▍ | 15099/44547 [5:06:52<7:24:56, 1.10it/s][2025-03-24 13:35:35] (step=0015100) Train Loss: 3.9733, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████▍ | 15124/44547 [5:07:15<7:23:48, 1.10it/s][2025-03-24 13:35:57] (step=0015125) Train Loss: 3.8895, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████▌ | 15149/44547 [5:07:37<7:23:41, 1.10it/s][2025-03-24 13:36:20] (step=0015150) Train Loss: 3.8821, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████▌ | 15174/44547 [5:08:00<7:24:13, 1.10it/s][2025-03-24 13:36:43] (step=0015175) Train Loss: 3.8190, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████▋ | 15199/44547 [5:08:23<7:23:31, 1.10it/s][2025-03-24 13:37:05] (step=0015200) Train Loss: 3.8423, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████▋ | 15224/44547 [5:08:45<7:26:15, 1.10it/s][2025-03-24 13:37:28] (step=0015225) Train Loss: 3.8995, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████▊ | 15249/44547 [5:09:08<7:23:33, 1.10it/s][2025-03-24 13:37:51] (step=0015250) Train Loss: 3.8992, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████▉ | 15274/44547 [5:09:31<7:22:16, 1.10it/s][2025-03-24 13:38:13] (step=0015275) Train Loss: 3.8707, Train Steps/Sec: 1.10 + 34%|███████████████████████████████████████████▉ | 15299/44547 [5:09:54<7:24:46, 1.10it/s][2025-03-24 13:38:36] (step=0015300) Train Loss: 3.8623, Train Steps/Sec: 1.10 + 34%|████████████████████████████████████████████ | 15324/44547 [5:10:16<7:23:32, 1.10it/s][2025-03-24 13:38:59] (step=0015325) Train Loss: 3.8247, Train Steps/Sec: 1.10 + 34%|████████████████████████████████████████████ | 15349/44547 [5:10:39<7:22:22, 1.10it/s][2025-03-24 13:39:22] (step=0015350) Train Loss: 3.8844, Train Steps/Sec: 1.10 + 35%|████████████████████████████████████████████▏ | 15374/44547 [5:11:03<7:20:41, 1.10it/s][2025-03-24 13:39:46] (step=0015375) Train Loss: 3.8235, Train Steps/Sec: 1.05 + 35%|████████████████████████████████████████████▏ | 15399/44547 [5:11:26<7:21:44, 1.10it/s][2025-03-24 13:40:08] (step=0015400) Train Loss: 3.8268, Train Steps/Sec: 1.10 + 35%|████████████████████████████████████████████▎ | 15424/44547 [5:11:48<7:20:15, 1.10it/s][2025-03-24 13:40:31] (step=0015425) Train Loss: 3.8349, Train Steps/Sec: 1.10 + 35%|████████████████████████████████████████████▍ | 15449/44547 [5:12:12<9:50:47, 1.22s/it][2025-03-24 13:40:55] (step=0015450) Train Loss: 3.8962, Train Steps/Sec: 1.05 + 35%|████████████████████████████████████████████▍ | 15474/44547 [5:12:36<7:35:13, 1.06it/s][2025-03-24 13:41:19] (step=0015475) Train Loss: 3.7685, Train Steps/Sec: 1.06 + 35%|████████████████████████████████████████████▌ | 15499/44547 [5:12:59<7:24:28, 1.09it/s][2025-03-24 13:41:42] (step=0015500) Train Loss: 3.9309, Train Steps/Sec: 1.06 + 35%|████████████████████████████████████████████▌ | 15524/44547 [5:13:22<7:19:28, 1.10it/s][2025-03-24 13:42:05] (step=0015525) Train Loss: 3.8653, Train Steps/Sec: 1.10 + 35%|████████████████████████████████████████████▋ | 15549/44547 [5:13:46<7:31:06, 1.07it/s][2025-03-24 13:42:29] (step=0015550) Train Loss: 3.8114, Train Steps/Sec: 1.05 + 35%|████████████████████████████████████████████▋ | 15574/44547 [5:14:09<7:17:40, 1.10it/s][2025-03-24 13:42:51] (step=0015575) Train Loss: 3.8429, Train Steps/Sec: 1.10 + 35%|████████████████████████████████████████████▊ | 15599/44547 [5:14:32<7:18:55, 1.10it/s][2025-03-24 13:43:14] (step=0015600) Train Loss: 3.7817, Train Steps/Sec: 1.10 + 35%|████████████████████████████████████████████▉ | 15624/44547 [5:14:54<7:18:10, 1.10it/s][2025-03-24 13:43:38] (step=0015625) Train Loss: 3.7790, Train Steps/Sec: 1.05 + 35%|████████████████████████████████████████████▉ | 15649/44547 [5:15:19<8:06:46, 1.01s/it][2025-03-24 13:44:02] (step=0015650) Train Loss: 3.7889, Train Steps/Sec: 1.06 + 35%|█████████████████████████████████████████████ | 15674/44547 [5:15:42<7:15:20, 1.11it/s][2025-03-24 13:44:24] (step=0015675) Train Loss: 4.0177, Train Steps/Sec: 1.10 + 35%|█████████████████████████████████████████████ | 15699/44547 [5:16:04<7:15:24, 1.10it/s][2025-03-24 13:44:47] (step=0015700) Train Loss: 3.8898, Train Steps/Sec: 1.10 + 35%|█████████████████████████████████████████████▏ | 15724/44547 [5:16:27<7:15:07, 1.10it/s][2025-03-24 13:45:10] (step=0015725) Train Loss: 3.9695, Train Steps/Sec: 1.10 + 35%|█████████████████████████████████████████████▎ | 15749/44547 [5:16:50<7:15:11, 1.10it/s][2025-03-24 13:45:32] (step=0015750) Train Loss: 3.9651, Train Steps/Sec: 1.10 + 35%|█████████████████████████████████████████████▎ | 15774/44547 [5:17:12<7:13:55, 1.11it/s][2025-03-24 13:45:55] (step=0015775) Train Loss: 3.7539, Train Steps/Sec: 1.10 + 35%|█████████████████████████████████████████████▍ | 15799/44547 [5:17:35<7:14:43, 1.10it/s][2025-03-24 13:46:18] (step=0015800) Train Loss: 3.9261, Train Steps/Sec: 1.10 + 36%|█████████████████████████████████████████████▍ | 15824/44547 [5:17:58<7:12:58, 1.11it/s][2025-03-24 13:46:40] (step=0015825) Train Loss: 3.7777, Train Steps/Sec: 1.10 + 36%|█████████████████████████████████████████████▌ | 15849/44547 [5:18:21<7:15:45, 1.10it/s][2025-03-24 13:47:04] (step=0015850) Train Loss: 3.9041, Train Steps/Sec: 1.07 + 36%|█████████████████████████████████████████████▌ | 15874/44547 [5:18:44<7:13:01, 1.10it/s][2025-03-24 13:47:27] (step=0015875) Train Loss: 3.8618, Train Steps/Sec: 1.10 + 36%|█████████████████████████████████████████████▋ | 15899/44547 [5:19:07<7:13:06, 1.10it/s][2025-03-24 13:47:49] (step=0015900) Train Loss: 3.8864, Train Steps/Sec: 1.10 + 36%|█████████████████████████████████████████████▊ | 15924/44547 [5:19:29<7:14:21, 1.10it/s][2025-03-24 13:48:12] (step=0015925) Train Loss: 3.8504, Train Steps/Sec: 1.10 + 36%|█████████████████████████████████████████████▊ | 15949/44547 [5:19:52<7:12:50, 1.10it/s][2025-03-24 13:48:35] (step=0015950) Train Loss: 3.9155, Train Steps/Sec: 1.10 + 36%|█████████████████████████████████████████████▉ | 15974/44547 [5:20:15<7:13:14, 1.10it/s][2025-03-24 13:48:58] (step=0015975) Train Loss: 3.9521, Train Steps/Sec: 1.10 + 36%|█████████████████████████████████████████████▉ | 15999/44547 [5:20:38<7:12:10, 1.10it/s][2025-03-24 13:49:20] (step=0016000) Train Loss: 3.9791, Train Steps/Sec: 1.10 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-03-24 13:49:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:56<00:00, 52.90s/it] +[2025-03-24 13:58:24] Finish Eval in 16000 steps...███████████████████████████████████████████████████████| 9/9 [07:55<00:00, 52.73s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-03-24 13:58:43] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0016000.pt +[2025-03-24 13:58:45] Removed old checkpoint: checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0012000.pt + 36%|██████████████████████████████████████████████ | 16024/44547 [5:30:25<7:28:02, 1.06it/s][2025-03-24 13:59:08] (step=0016025) Train Loss: 3.8066, Train Steps/Sec: 0.04 + 36%|██████████████████████████████████████████████ | 16049/44547 [5:30:48<7:13:31, 1.10it/s][2025-03-24 13:59:31] (step=0016050) Train Loss: 3.9557, Train Steps/Sec: 1.10 + 36%|██████████████████████████████████████████████▏ | 16074/44547 [5:31:11<7:10:53, 1.10it/s][2025-03-24 13:59:54] (step=0016075) Train Loss: 3.9360, Train Steps/Sec: 1.10 + 36%|██████████████████████████████████████████████▎ | 16099/44547 [5:31:34<7:11:58, 1.10it/s][2025-03-24 14:00:16] (step=0016100) Train Loss: 3.9689, Train Steps/Sec: 1.10 + 36%|██████████████████████████████████████████████▎ | 16124/44547 [5:31:56<7:10:00, 1.10it/s][2025-03-24 14:00:39] (step=0016125) Train Loss: 3.7709, Train Steps/Sec: 1.10 + 36%|██████████████████████████████████████████████▍ | 16149/44547 [5:32:20<7:10:25, 1.10it/s][2025-03-24 14:01:03] (step=0016150) Train Loss: 3.7712, Train Steps/Sec: 1.06 + 36%|██████████████████████████████████████████████▍ | 16174/44547 [5:32:45<7:09:56, 1.10it/s][2025-03-24 14:01:29] (step=0016175) Train Loss: 3.8381, Train Steps/Sec: 0.97 + 36%|██████████████████████████████████████████████▌ | 16199/44547 [5:33:09<7:16:11, 1.08it/s][2025-03-24 14:01:51] (step=0016200) Train Loss: 3.8382, Train Steps/Sec: 1.09 + 36%|██████████████████████████████████████████████▌ | 16224/44547 [5:33:33<7:17:12, 1.08it/s][2025-03-24 14:02:15] (step=0016225) Train Loss: 3.9317, Train Steps/Sec: 1.05 + 36%|██████████████████████████████████████████████▋ | 16249/44547 [5:33:56<7:07:07, 1.10it/s][2025-03-24 14:02:39] (step=0016250) Train Loss: 3.8412, Train Steps/Sec: 1.06 + 37%|██████████████████████████████████████████████▊ | 16274/44547 [5:34:19<7:05:48, 1.11it/s][2025-03-24 14:03:01] (step=0016275) Train Loss: 3.8575, Train Steps/Sec: 1.10 + 37%|██████████████████████████████████████████████▊ | 16299/44547 [5:34:41<7:05:48, 1.11it/s][2025-03-24 14:03:24] (step=0016300) Train Loss: 3.8671, Train Steps/Sec: 1.10 + 37%|██████████████████████████████████████████████▉ | 16324/44547 [5:35:04<7:06:19, 1.10it/s][2025-03-24 14:03:47] (step=0016325) Train Loss: 3.8698, Train Steps/Sec: 1.10 + 37%|██████████████████████████████████████████████▉ | 16349/44547 [5:35:27<7:05:07, 1.11it/s][2025-03-24 14:04:09] (step=0016350) Train Loss: 3.8967, Train Steps/Sec: 1.10 + 37%|███████████████████████████████████████████████ | 16374/44547 [5:35:49<7:05:10, 1.10it/s][2025-03-24 14:04:32] (step=0016375) Train Loss: 3.8295, Train Steps/Sec: 1.10 + 37%|███████████████████████████████████████████████ | 16399/44547 [5:36:12<7:05:42, 1.10it/s][2025-03-24 14:04:55] (step=0016400) Train Loss: 3.9397, Train Steps/Sec: 1.10 + 37%|███████████████████████████████████████████████▏ | 16424/44547 [5:36:35<7:04:34, 1.10it/s][2025-03-24 14:05:17] (step=0016425) Train Loss: 3.8931, Train Steps/Sec: 1.10 + 37%|███████████████████████████████████████████████▎ | 16449/44547 [5:36:58<8:08:08, 1.04s/it][2025-03-24 14:05:41] (step=0016450) Train Loss: 3.8986, Train Steps/Sec: 1.07 + 37%|███████████████████████████████████████████████▎ | 16474/44547 [5:37:21<7:05:10, 1.10it/s][2025-03-24 14:06:03] (step=0016475) Train Loss: 3.8616, Train Steps/Sec: 1.10 + 37%|███████████████████████████████████████████████▍ | 16499/44547 [5:37:44<7:02:51, 1.11it/s][2025-03-24 14:06:26] (step=0016500) Train Loss: 3.8681, Train Steps/Sec: 1.10 + 37%|███████████████████████████████████████████████▍ | 16524/44547 [5:38:06<7:02:54, 1.10it/s][2025-03-24 14:06:49] (step=0016525) Train Loss: 3.7683, Train Steps/Sec: 1.10 + 37%|███████████████████████████████████████████████▌ | 16549/44547 [5:38:29<7:03:30, 1.10it/s][2025-03-24 14:07:12] (step=0016550) Train Loss: 3.8334, Train Steps/Sec: 1.10 + 37%|███████████████████████████████████████████████▌ | 16574/44547 [5:38:53<7:28:21, 1.04it/s][2025-03-24 14:07:35] (step=0016575) Train Loss: 3.8202, Train Steps/Sec: 1.05 + 37%|███████████████████████████████████████████████▋ | 16599/44547 [5:39:17<7:56:20, 1.02s/it][2025-03-24 14:07:59] (step=0016600) Train Loss: 3.8610, Train Steps/Sec: 1.05 + 37%|███████████████████████████████████████████████▊ | 16624/44547 [5:39:39<7:01:24, 1.10it/s][2025-03-24 14:08:22] (step=0016625) Train Loss: 3.9381, Train Steps/Sec: 1.10 + 37%|███████████████████████████████████████████████▊ | 16649/44547 [5:40:02<7:01:51, 1.10it/s][2025-03-24 14:08:45] (step=0016650) Train Loss: 3.8912, Train Steps/Sec: 1.10 + 37%|███████████████████████████████████████████████▉ | 16674/44547 [5:40:25<7:00:32, 1.10it/s][2025-03-24 14:09:07] (step=0016675) Train Loss: 3.7935, Train Steps/Sec: 1.10 + 37%|███████████████████████████████████████████████▉ | 16699/44547 [5:40:47<7:01:58, 1.10it/s][2025-03-24 14:09:30] (step=0016700) Train Loss: 3.9022, Train Steps/Sec: 1.10 + 38%|████████████████████████████████████████████████ | 16724/44547 [5:41:11<7:11:56, 1.07it/s][2025-03-24 14:09:54] (step=0016725) Train Loss: 3.7605, Train Steps/Sec: 1.05 + 38%|████████████████████████████████████████████████▏ | 16749/44547 [5:41:34<7:00:08, 1.10it/s][2025-03-24 14:10:17] (step=0016750) Train Loss: 3.7745, Train Steps/Sec: 1.10 + 38%|████████████████████████████████████████████████▏ | 16774/44547 [5:41:57<7:00:29, 1.10it/s][2025-03-24 14:10:39] (step=0016775) Train Loss: 3.9360, Train Steps/Sec: 1.10 + 38%|████████████████████████████████████████████████▎ | 16799/44547 [5:42:19<6:59:32, 1.10it/s][2025-03-24 14:11:02] (step=0016800) Train Loss: 3.8303, Train Steps/Sec: 1.10 + 38%|████████████████████████████████████████████████▎ | 16824/44547 [5:42:42<6:58:07, 1.11it/s][2025-03-24 14:11:25] (step=0016825) Train Loss: 3.8536, Train Steps/Sec: 1.10 + 38%|████████████████████████████████████████████████▍ | 16849/44547 [5:43:05<7:01:51, 1.09it/s][2025-03-24 14:11:48] (step=0016850) Train Loss: 3.7777, Train Steps/Sec: 1.07 + 38%|████████████████████████████████████████████████▍ | 16874/44547 [5:43:28<6:59:36, 1.10it/s][2025-03-24 14:12:11] (step=0016875) Train Loss: 3.8617, Train Steps/Sec: 1.10 + 38%|████████████████████████████████████████████████▌ | 16899/44547 [5:43:51<6:57:14, 1.10it/s][2025-03-24 14:12:34] (step=0016900) Train Loss: 3.9413, Train Steps/Sec: 1.10 + 38%|████████████████████████████████████████████████▋ | 16924/44547 [5:44:14<6:58:30, 1.10it/s][2025-03-24 14:12:56] (step=0016925) Train Loss: 3.8253, Train Steps/Sec: 1.10 + 38%|████████████████████████████████████████████████▋ | 16949/44547 [5:44:36<6:55:40, 1.11it/s][2025-03-24 14:13:19] (step=0016950) Train Loss: 3.8193, Train Steps/Sec: 1.10 + 38%|████████████████████████████████████████████████▊ | 16974/44547 [5:45:00<6:59:34, 1.10it/s][2025-03-24 14:13:43] (step=0016975) Train Loss: 3.9212, Train Steps/Sec: 1.06 + 38%|████████████████████████████████████████████████▊ | 16999/44547 [5:45:25<7:06:21, 1.08it/s][2025-03-24 14:14:07] (step=0017000) Train Loss: 3.7602, Train Steps/Sec: 1.02 + 38%|████████████████████████████████████████████████▉ | 17024/44547 [5:45:48<6:57:42, 1.10it/s][2025-03-24 14:14:31] (step=0017025) Train Loss: 3.9568, Train Steps/Sec: 1.05 + 38%|████████████████████████████████████████████████▉ | 17049/44547 [5:46:11<6:58:36, 1.09it/s][2025-03-24 14:14:54] (step=0017050) Train Loss: 3.8706, Train Steps/Sec: 1.10 + 38%|█████████████████████████████████████████████████ | 17074/44547 [5:46:34<6:56:36, 1.10it/s][2025-03-24 14:15:16] (step=0017075) Train Loss: 3.8716, Train Steps/Sec: 1.10 + 38%|█████████████████████████████████████████████████▏ | 17099/44547 [5:46:56<6:56:04, 1.10it/s][2025-03-24 14:15:39] (step=0017100) Train Loss: 3.9352, Train Steps/Sec: 1.10 + 38%|█████████████████████████████████████████████████▏ | 17124/44547 [5:47:19<6:55:34, 1.10it/s][2025-03-24 14:16:02] (step=0017125) Train Loss: 3.8118, Train Steps/Sec: 1.10 + 38%|█████████████████████████████████████████████████▎ | 17149/44547 [5:47:42<6:54:56, 1.10it/s][2025-03-24 14:16:25] (step=0017150) Train Loss: 3.8913, Train Steps/Sec: 1.10 + 39%|█████████████████████████████████████████████████▎ | 17174/44547 [5:48:06<6:54:27, 1.10it/s][2025-03-24 14:16:48] (step=0017175) Train Loss: 3.7869, Train Steps/Sec: 1.05 + 39%|█████████████████████████████████████████████████▍ | 17199/44547 [5:48:28<6:53:08, 1.10it/s][2025-03-24 14:17:11] (step=0017200) Train Loss: 3.8789, Train Steps/Sec: 1.10 + 39%|█████████████████████████████████████████████████▍ | 17224/44547 [5:48:52<6:54:27, 1.10it/s][2025-03-24 14:17:35] (step=0017225) Train Loss: 3.9886, Train Steps/Sec: 1.06 + 39%|█████████████████████████████████████████████████▌ | 17249/44547 [5:49:15<6:52:22, 1.10it/s][2025-03-24 14:17:57] (step=0017250) Train Loss: 3.8319, Train Steps/Sec: 1.10 + 39%|█████████████████████████████████████████████████▋ | 17274/44547 [5:49:38<6:53:28, 1.10it/s][2025-03-24 14:18:20] (step=0017275) Train Loss: 3.7174, Train Steps/Sec: 1.10 + 39%|█████████████████████████████████████████████████▋ | 17299/44547 [5:50:00<6:52:15, 1.10it/s][2025-03-24 14:18:43] (step=0017300) Train Loss: 3.8653, Train Steps/Sec: 1.10 + 39%|█████████████████████████████████████████████████▊ | 17324/44547 [5:50:23<6:52:08, 1.10it/s][2025-03-24 14:19:06] (step=0017325) Train Loss: 3.8260, Train Steps/Sec: 1.10 + 39%|█████████████████████████████████████████████████▊ | 17349/44547 [5:50:46<6:52:54, 1.10it/s][2025-03-24 14:19:28] (step=0017350) Train Loss: 3.9005, Train Steps/Sec: 1.10 + 39%|█████████████████████████████████████████████████▉ | 17374/44547 [5:51:08<6:51:13, 1.10it/s][2025-03-24 14:19:51] (step=0017375) Train Loss: 3.8592, Train Steps/Sec: 1.10 + 39%|█████████████████████████████████████████████████▉ | 17399/44547 [5:51:31<6:54:21, 1.09it/s][2025-03-24 14:20:14] (step=0017400) Train Loss: 3.8639, Train Steps/Sec: 1.10 + 39%|██████████████████████████████████████████████████ | 17424/44547 [5:51:54<6:50:14, 1.10it/s][2025-03-24 14:20:37] (step=0017425) Train Loss: 3.9175, Train Steps/Sec: 1.10 + 39%|██████████████████████████████████████████████████▏ | 17449/44547 [5:52:17<6:52:15, 1.10it/s][2025-03-24 14:20:59] (step=0017450) Train Loss: 3.8503, Train Steps/Sec: 1.10 + 39%|██████████████████████████████████████████████████▏ | 17474/44547 [5:52:39<6:50:26, 1.10it/s][2025-03-24 14:21:22] (step=0017475) Train Loss: 3.8219, Train Steps/Sec: 1.10 + 39%|██████████████████████████████████████████████████▎ | 17499/44547 [5:53:03<6:50:59, 1.10it/s][2025-03-24 14:21:46] (step=0017500) Train Loss: 3.7731, Train Steps/Sec: 1.05 + 39%|██████████████████████████████████████████████████▎ | 17524/44547 [5:53:28<7:55:12, 1.06s/it][2025-03-24 14:22:11] (step=0017525) Train Loss: 3.7650, Train Steps/Sec: 1.01 + 39%|██████████████████████████████████████████████████▍ | 17549/44547 [5:53:51<6:47:15, 1.10it/s][2025-03-24 14:22:33] (step=0017550) Train Loss: 3.9273, Train Steps/Sec: 1.10 + 39%|██████████████████████████████████████████████████▍ | 17574/44547 [5:54:13<6:46:56, 1.10it/s][2025-03-24 14:22:56] (step=0017575) Train Loss: 3.9033, Train Steps/Sec: 1.10 + 40%|██████████████████████████████████████████████████▌ | 17599/44547 [5:54:36<6:49:24, 1.10it/s][2025-03-24 14:23:19] (step=0017600) Train Loss: 3.8509, Train Steps/Sec: 1.10 + 40%|██████████████████████████████████████████████████▋ | 17624/44547 [5:54:59<6:47:45, 1.10it/s][2025-03-24 14:23:41] (step=0017625) Train Loss: 3.8548, Train Steps/Sec: 1.10 + 40%|██████████████████████████████████████████████████▋ | 17649/44547 [5:55:21<6:48:09, 1.10it/s][2025-03-24 14:24:04] (step=0017650) Train Loss: 3.8728, Train Steps/Sec: 1.10 + 40%|██████████████████████████████████████████████████▊ | 17674/44547 [5:55:44<6:46:01, 1.10it/s][2025-03-24 14:24:27] (step=0017675) Train Loss: 3.8869, Train Steps/Sec: 1.10 + 40%|██████████████████████████████████████████████████▊ | 17699/44547 [5:56:09<9:21:37, 1.26s/it][2025-03-24 14:24:52] (step=0017700) Train Loss: 3.8947, Train Steps/Sec: 1.00 + 40%|██████████████████████████████████████████████████▉ | 17724/44547 [5:56:33<8:49:48, 1.19s/it][2025-03-24 14:25:15] (step=0017725) Train Loss: 3.8783, Train Steps/Sec: 1.06 + 40%|██████████████████████████████████████████████████▉ | 17749/44547 [5:56:56<6:43:41, 1.11it/s][2025-03-24 14:25:39] (step=0017750) Train Loss: 3.8678, Train Steps/Sec: 1.06 + 40%|███████████████████████████████████████████████████ | 17774/44547 [5:57:20<7:00:43, 1.06it/s][2025-03-24 14:26:03] (step=0017775) Train Loss: 3.9771, Train Steps/Sec: 1.06 + 40%|███████████████████████████████████████████████████▏ | 17799/44547 [5:57:43<6:45:23, 1.10it/s][2025-03-24 14:26:25] (step=0017800) Train Loss: 3.8506, Train Steps/Sec: 1.10 + 40%|███████████████████████████████████████████████████▏ | 17824/44547 [5:58:05<6:42:57, 1.11it/s][2025-03-24 14:26:48] (step=0017825) Train Loss: 3.7639, Train Steps/Sec: 1.10 + 40%|███████████████████████████████████████████████████▎ | 17849/44547 [5:58:28<6:42:48, 1.10it/s][2025-03-24 14:27:11] (step=0017850) Train Loss: 3.8348, Train Steps/Sec: 1.10 + 40%|███████████████████████████████████████████████████▎ | 17874/44547 [5:58:51<6:41:42, 1.11it/s][2025-03-24 14:27:33] (step=0017875) Train Loss: 3.6688, Train Steps/Sec: 1.10 + 40%|███████████████████████████████████████████████████▍ | 17899/44547 [5:59:13<6:41:45, 1.11it/s][2025-03-24 14:27:56] (step=0017900) Train Loss: 3.8955, Train Steps/Sec: 1.10 + 40%|███████████████████████████████████████████████████▌ | 17924/44547 [5:59:36<6:42:20, 1.10it/s][2025-03-24 14:28:19] (step=0017925) Train Loss: 3.8874, Train Steps/Sec: 1.10 + 40%|███████████████████████████████████████████████████▌ | 17949/44547 [5:59:59<6:42:00, 1.10it/s][2025-03-24 14:28:41] (step=0017950) Train Loss: 3.8180, Train Steps/Sec: 1.10 + 40%|███████████████████████████████████████████████████▋ | 17974/44547 [6:00:22<6:43:35, 1.10it/s][2025-03-24 14:29:05] (step=0017975) Train Loss: 3.8059, Train Steps/Sec: 1.05 + 40%|███████████████████████████████████████████████████▋ | 17999/44547 [6:00:45<6:42:13, 1.10it/s][2025-03-24 14:29:28] (step=0018000) Train Loss: 3.8513, Train Steps/Sec: 1.10 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-03-24 14:29:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:57<00:00, 53.11s/it] +[2025-03-24 14:38:35] Finish Eval in 18000 steps...███████████████████████████████████████████████████████| 9/9 [07:57<00:00, 52.91s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... + 40%|███████████████████████████████████████████████████▊ | 18024/44547 [6:10:15<6:56:30, 1.06it/s][2025-03-24 14:38:58] (step=0018025) Train Loss: 3.7996, Train Steps/Sec: 0.04 + 41%|███████████████████████████████████████████████████▊ | 18049/44547 [6:10:38<6:42:36, 1.10it/s][2025-03-24 14:39:21] (step=0018050) Train Loss: 3.8961, Train Steps/Sec: 1.10 + 41%|███████████████████████████████████████████████████▉ | 18074/44547 [6:11:01<6:39:25, 1.10it/s][2025-03-24 14:39:44] (step=0018075) Train Loss: 3.7416, Train Steps/Sec: 1.09 + 41%|████████████████████████████████████████████████████ | 18099/44547 [6:11:24<6:39:55, 1.10it/s][2025-03-24 14:40:06] (step=0018100) Train Loss: 3.7171, Train Steps/Sec: 1.10 + 41%|████████████████████████████████████████████████████ | 18124/44547 [6:11:46<6:40:01, 1.10it/s][2025-03-24 14:40:29] (step=0018125) Train Loss: 3.8075, Train Steps/Sec: 1.10 + 41%|████████████████████████████████████████████████████▏ | 18149/44547 [6:12:09<6:39:57, 1.10it/s][2025-03-24 14:40:52] (step=0018150) Train Loss: 3.9347, Train Steps/Sec: 1.10 + 41%|████████████████████████████████████████████████████▏ | 18174/44547 [6:12:32<6:39:04, 1.10it/s][2025-03-24 14:41:15] (step=0018175) Train Loss: 3.8404, Train Steps/Sec: 1.10 + 41%|████████████████████████████████████████████████████▎ | 18199/44547 [6:12:55<6:38:13, 1.10it/s][2025-03-24 14:41:37] (step=0018200) Train Loss: 3.8659, Train Steps/Sec: 1.10 + 41%|████████████████████████████████████████████████████▎ | 18224/44547 [6:13:18<6:40:38, 1.10it/s][2025-03-24 14:42:01] (step=0018225) Train Loss: 3.7767, Train Steps/Sec: 1.07 + 41%|████████████████████████████████████████████████████▍ | 18249/44547 [6:13:41<6:38:28, 1.10it/s][2025-03-24 14:42:23] (step=0018250) Train Loss: 3.7942, Train Steps/Sec: 1.10 + 41%|████████████████████████████████████████████████████▌ | 18274/44547 [6:14:04<6:37:13, 1.10it/s][2025-03-24 14:42:47] (step=0018275) Train Loss: 3.7771, Train Steps/Sec: 1.06 + 41%|████████████████████████████████████████████████████▌ | 18299/44547 [6:14:27<6:37:03, 1.10it/s][2025-03-24 14:43:10] (step=0018300) Train Loss: 3.8449, Train Steps/Sec: 1.10 + 41%|████████████████████████████████████████████████████▋ | 18324/44547 [6:14:51<8:14:11, 1.13s/it][2025-03-24 14:43:33] (step=0018325) Train Loss: 3.8579, Train Steps/Sec: 1.05 + 41%|████████████████████████████████████████████████████▋ | 18349/44547 [6:15:13<6:36:38, 1.10it/s][2025-03-24 14:43:56] (step=0018350) Train Loss: 3.8469, Train Steps/Sec: 1.10 + 41%|████████████████████████████████████████████████████▊ | 18374/44547 [6:15:36<6:34:55, 1.10it/s][2025-03-24 14:44:19] (step=0018375) Train Loss: 3.8078, Train Steps/Sec: 1.10 + 41%|████████████████████████████████████████████████████▊ | 18399/44547 [6:16:00<6:35:24, 1.10it/s][2025-03-24 14:44:43] (step=0018400) Train Loss: 3.7304, Train Steps/Sec: 1.05 + 41%|████████████████████████████████████████████████████▉ | 18424/44547 [6:16:23<6:34:05, 1.10it/s][2025-03-24 14:45:06] (step=0018425) Train Loss: 3.7395, Train Steps/Sec: 1.10 + 41%|█████████████████████████████████████████████████████ | 18449/44547 [6:16:47<6:33:47, 1.10it/s][2025-03-24 14:45:29] (step=0018450) Train Loss: 3.9243, Train Steps/Sec: 1.10 + 41%|█████████████████████████████████████████████████████ | 18474/44547 [6:17:09<6:33:33, 1.10it/s][2025-03-24 14:45:52] (step=0018475) Train Loss: 3.8670, Train Steps/Sec: 1.11 + 42%|█████████████████████████████████████████████████████▏ | 18499/44547 [6:17:32<6:34:00, 1.10it/s][2025-03-24 14:46:14] (step=0018500) Train Loss: 3.8429, Train Steps/Sec: 1.10 + 42%|█████████████████████████████████████████████████████▏ | 18524/44547 [6:17:55<6:32:43, 1.10it/s][2025-03-24 14:46:37] (step=0018525) Train Loss: 3.9065, Train Steps/Sec: 1.10 + 42%|█████████████████████████████████████████████████████▎ | 18549/44547 [6:18:18<6:32:45, 1.10it/s][2025-03-24 14:47:01] (step=0018550) Train Loss: 3.9229, Train Steps/Sec: 1.06 + 42%|█████████████████████████████████████████████████████▎ | 18574/44547 [6:18:42<6:33:06, 1.10it/s][2025-03-24 14:47:24] (step=0018575) Train Loss: 3.7993, Train Steps/Sec: 1.05 + 42%|█████████████████████████████████████████████████████▍ | 18599/44547 [6:19:05<6:31:57, 1.10it/s][2025-03-24 14:47:47] (step=0018600) Train Loss: 3.7361, Train Steps/Sec: 1.10 + 42%|█████████████████████████████████████████████████████▌ | 18624/44547 [6:19:27<6:33:24, 1.10it/s][2025-03-24 14:48:10] (step=0018625) Train Loss: 3.7571, Train Steps/Sec: 1.10 + 42%|█████████████████████████████████████████████████████▌ | 18649/44547 [6:19:50<6:31:45, 1.10it/s][2025-03-24 14:48:33] (step=0018650) Train Loss: 3.7463, Train Steps/Sec: 1.10 + 42%|█████████████████████████████████████████████████████▋ | 18674/44547 [6:20:13<6:29:24, 1.11it/s][2025-03-24 14:48:55] (step=0018675) Train Loss: 3.7820, Train Steps/Sec: 1.10 + 42%|█████████████████████████████████████████████████████▋ | 18699/44547 [6:20:36<9:06:51, 1.27s/it][2025-03-24 14:49:19] (step=0018700) Train Loss: 3.8422, Train Steps/Sec: 1.04 + 42%|█████████████████████████████████████████████████████▊ | 18724/44547 [6:20:59<6:32:49, 1.10it/s][2025-03-24 14:49:42] (step=0018725) Train Loss: 3.8843, Train Steps/Sec: 1.10 + 42%|█████████████████████████████████████████████████████▊ | 18749/44547 [6:21:22<6:30:48, 1.10it/s][2025-03-24 14:50:05] (step=0018750) Train Loss: 3.8065, Train Steps/Sec: 1.10 + 42%|█████████████████████████████████████████████████████▉ | 18774/44547 [6:21:45<6:30:07, 1.10it/s][2025-03-24 14:50:27] (step=0018775) Train Loss: 3.8358, Train Steps/Sec: 1.10 + 42%|██████████████████████████████████████████████████████ | 18799/44547 [6:22:08<6:30:18, 1.10it/s][2025-03-24 14:50:50] (step=0018800) Train Loss: 3.7009, Train Steps/Sec: 1.10 + 42%|██████████████████████████████████████████████████████ | 18824/44547 [6:22:30<6:29:25, 1.10it/s][2025-03-24 14:51:13] (step=0018825) Train Loss: 3.7547, Train Steps/Sec: 1.10 + 42%|██████████████████████████████████████████████████████▏ | 18849/44547 [6:22:53<6:27:32, 1.11it/s][2025-03-24 14:51:36] (step=0018850) Train Loss: 3.8871, Train Steps/Sec: 1.10 + 42%|██████████████████████████████████████████████████████▏ | 18874/44547 [6:23:16<6:28:35, 1.10it/s][2025-03-24 14:51:58] (step=0018875) Train Loss: 3.7758, Train Steps/Sec: 1.10 + 42%|██████████████████████████████████████████████████████▎ | 18899/44547 [6:23:38<6:29:36, 1.10it/s][2025-03-24 14:52:21] (step=0018900) Train Loss: 3.8429, Train Steps/Sec: 1.10 + 42%|██████████████████████████████████████████████████████▍ | 18924/44547 [6:24:03<6:48:04, 1.05it/s][2025-03-24 14:52:46] (step=0018925) Train Loss: 3.8349, Train Steps/Sec: 1.03 + 43%|██████████████████████████████████████████████████████▍ | 18949/44547 [6:24:26<6:26:24, 1.10it/s][2025-03-24 14:53:09] (step=0018950) Train Loss: 3.8328, Train Steps/Sec: 1.10 + 43%|██████████████████████████████████████████████████████▌ | 18974/44547 [6:24:50<6:28:01, 1.10it/s][2025-03-24 14:53:33] (step=0018975) Train Loss: 3.8681, Train Steps/Sec: 1.06 + 43%|██████████████████████████████████████████████████████▌ | 18999/44547 [6:25:13<6:25:07, 1.11it/s][2025-03-24 14:53:55] (step=0019000) Train Loss: 3.7820, Train Steps/Sec: 1.10 + 43%|██████████████████████████████████████████████████████▋ | 19024/44547 [6:25:36<6:26:07, 1.10it/s][2025-03-24 14:54:18] (step=0019025) Train Loss: 3.8375, Train Steps/Sec: 1.10 + 43%|██████████████████████████████████████████████████████▋ | 19049/44547 [6:25:58<6:24:32, 1.11it/s][2025-03-24 14:54:41] (step=0019050) Train Loss: 3.8439, Train Steps/Sec: 1.10 + 43%|██████████████████████████████████████████████████████▊ | 19074/44547 [6:26:21<6:28:04, 1.09it/s][2025-03-24 14:55:04] (step=0019075) Train Loss: 3.7178, Train Steps/Sec: 1.10 + 43%|██████████████████████████████████████████████████████▉ | 19099/44547 [6:26:44<6:24:55, 1.10it/s][2025-03-24 14:55:26] (step=0019100) Train Loss: 3.7700, Train Steps/Sec: 1.10 + 43%|██████████████████████████████████████████████████████▉ | 19124/44547 [6:27:07<6:55:53, 1.02it/s][2025-03-24 14:55:50] (step=0019125) Train Loss: 3.8169, Train Steps/Sec: 1.05 + 43%|███████████████████████████████████████████████████████ | 19149/44547 [6:27:31<6:29:14, 1.09it/s][2025-03-24 14:56:14] (step=0019150) Train Loss: 3.8558, Train Steps/Sec: 1.05 + 43%|███████████████████████████████████████████████████████ | 19174/44547 [6:27:55<7:20:38, 1.04s/it][2025-03-24 14:56:37] (step=0019175) Train Loss: 3.9177, Train Steps/Sec: 1.06 + 43%|███████████████████████████████████████████████████████▏ | 19199/44547 [6:28:17<6:22:46, 1.10it/s][2025-03-24 14:57:00] (step=0019200) Train Loss: 3.7491, Train Steps/Sec: 1.10 + 43%|███████████████████████████████████████████████████████▏ | 19224/44547 [6:28:40<6:21:28, 1.11it/s][2025-03-24 14:57:23] (step=0019225) Train Loss: 3.8615, Train Steps/Sec: 1.10 + 43%|███████████████████████████████████████████████████████▎ | 19249/44547 [6:29:03<6:25:10, 1.09it/s][2025-03-24 14:57:45] (step=0019250) Train Loss: 3.9140, Train Steps/Sec: 1.10 + 43%|███████████████████████████████████████████████████████▍ | 19274/44547 [6:29:25<6:24:31, 1.10it/s][2025-03-24 14:58:08] (step=0019275) Train Loss: 3.8395, Train Steps/Sec: 1.10 + 43%|███████████████████████████████████████████████████████▍ | 19299/44547 [6:29:48<6:21:53, 1.10it/s][2025-03-24 14:58:31] (step=0019300) Train Loss: 3.8726, Train Steps/Sec: 1.10 + 43%|███████████████████████████████████████████████████████▌ | 19324/44547 [6:30:11<6:20:41, 1.10it/s][2025-03-24 14:58:53] (step=0019325) Train Loss: 3.8306, Train Steps/Sec: 1.10 + 43%|███████████████████████████████████████████████████████▌ | 19349/44547 [6:30:33<6:22:07, 1.10it/s][2025-03-24 14:59:16] (step=0019350) Train Loss: 3.7320, Train Steps/Sec: 1.10 + 43%|███████████████████████████████████████████████████████▋ | 19374/44547 [6:30:56<6:21:23, 1.10it/s][2025-03-24 14:59:39] (step=0019375) Train Loss: 3.7602, Train Steps/Sec: 1.10 + 44%|███████████████████████████████████████████████████████▋ | 19399/44547 [6:31:21<7:18:56, 1.05s/it][2025-03-24 15:00:03] (step=0019400) Train Loss: 3.8010, Train Steps/Sec: 1.02 + 44%|███████████████████████████████████████████████████████▊ | 19424/44547 [6:31:43<6:20:07, 1.10it/s][2025-03-24 15:00:26] (step=0019425) Train Loss: 3.7561, Train Steps/Sec: 1.10 + 44%|███████████████████████████████████████████████████████▉ | 19449/44547 [6:32:07<7:13:04, 1.04s/it][2025-03-24 15:00:50] (step=0019450) Train Loss: 3.9424, Train Steps/Sec: 1.05 + 44%|███████████████████████████████████████████████████████▉ | 19474/44547 [6:32:30<6:18:39, 1.10it/s][2025-03-24 15:01:13] (step=0019475) Train Loss: 3.7782, Train Steps/Sec: 1.10 + 44%|████████████████████████████████████████████████████████ | 19499/44547 [6:32:53<6:19:58, 1.10it/s][2025-03-24 15:01:35] (step=0019500) Train Loss: 3.7318, Train Steps/Sec: 1.10 + 44%|████████████████████████████████████████████████████████ | 19524/44547 [6:33:16<6:18:51, 1.10it/s][2025-03-24 15:01:59] (step=0019525) Train Loss: 3.9102, Train Steps/Sec: 1.05 + 44%|████████████████████████████████████████████████████████▏ | 19549/44547 [6:33:39<6:19:48, 1.10it/s][2025-03-24 15:02:22] (step=0019550) Train Loss: 3.8321, Train Steps/Sec: 1.10 + 44%|████████████████████████████████████████████████████████▏ | 19574/44547 [6:34:02<6:16:58, 1.10it/s][2025-03-24 15:02:45] (step=0019575) Train Loss: 3.7027, Train Steps/Sec: 1.10 + 44%|████████████████████████████████████████████████████████▎ | 19599/44547 [6:34:25<6:19:16, 1.10it/s][2025-03-24 15:03:07] (step=0019600) Train Loss: 3.7694, Train Steps/Sec: 1.10 + 44%|████████████████████████████████████████████████████████▍ | 19624/44547 [6:34:47<6:17:15, 1.10it/s][2025-03-24 15:03:30] (step=0019625) Train Loss: 3.7614, Train Steps/Sec: 1.10 + 44%|████████████████████████████████████████████████████████▍ | 19649/44547 [6:35:10<6:17:26, 1.10it/s][2025-03-24 15:03:53] (step=0019650) Train Loss: 3.7758, Train Steps/Sec: 1.10 + 44%|████████████████████████████████████████████████████████▌ | 19674/44547 [6:35:33<6:15:36, 1.10it/s][2025-03-24 15:04:15] (step=0019675) Train Loss: 3.7746, Train Steps/Sec: 1.10 + 44%|████████████████████████████████████████████████████████▌ | 19699/44547 [6:35:55<6:14:56, 1.10it/s][2025-03-24 15:04:38] (step=0019700) Train Loss: 3.8159, Train Steps/Sec: 1.10 + 44%|████████████████████████████████████████████████████████▋ | 19724/44547 [6:36:18<6:13:04, 1.11it/s][2025-03-24 15:05:02] (step=0019725) Train Loss: 3.7827, Train Steps/Sec: 1.06 + 44%|████████████████████████████████████████████████████████▋ | 19749/44547 [6:36:42<6:16:51, 1.10it/s][2025-03-24 15:05:24] (step=0019750) Train Loss: 3.7938, Train Steps/Sec: 1.10 + 44%|████████████████████████████████████████████████████████▊ | 19774/44547 [6:37:05<6:14:46, 1.10it/s][2025-03-24 15:05:47] (step=0019775) Train Loss: 3.7209, Train Steps/Sec: 1.10 + 44%|████████████████████████████████████████████████████████▉ | 19799/44547 [6:37:28<6:55:45, 1.01s/it][2025-03-24 15:06:11] (step=0019800) Train Loss: 3.8526, Train Steps/Sec: 1.06 + 45%|████████████████████████████████████████████████████████▉ | 19824/44547 [6:37:52<6:14:49, 1.10it/s][2025-03-24 15:06:35] (step=0019825) Train Loss: 3.8777, Train Steps/Sec: 1.06 + 45%|█████████████████████████████████████████████████████████ | 19849/44547 [6:38:16<6:13:23, 1.10it/s][2025-03-24 15:06:59] (step=0019850) Train Loss: 3.7975, Train Steps/Sec: 1.03 + 45%|█████████████████████████████████████████████████████████ | 19874/44547 [6:38:39<6:13:15, 1.10it/s][2025-03-24 15:07:21] (step=0019875) Train Loss: 3.9378, Train Steps/Sec: 1.10 + 45%|█████████████████████████████████████████████████████████▏ | 19899/44547 [6:39:02<6:13:01, 1.10it/s][2025-03-24 15:07:44] (step=0019900) Train Loss: 3.8433, Train Steps/Sec: 1.10 + 45%|█████████████████████████████████████████████████████████▏ | 19924/44547 [6:39:24<6:13:18, 1.10it/s][2025-03-24 15:08:07] (step=0019925) Train Loss: 3.8253, Train Steps/Sec: 1.10 + 45%|█████████████████████████████████████████████████████████▎ | 19949/44547 [6:39:48<6:29:30, 1.05it/s][2025-03-24 15:08:31] (step=0019950) Train Loss: 3.7303, Train Steps/Sec: 1.04 + 45%|█████████████████████████████████████████████████████████▍ | 19974/44547 [6:40:11<6:11:54, 1.10it/s][2025-03-24 15:08:54] (step=0019975) Train Loss: 3.8406, Train Steps/Sec: 1.10 + 45%|█████████████████████████████████████████████████████████▍ | 19999/44547 [6:40:34<6:10:10, 1.11it/s][2025-03-24 15:09:16] (step=0020000) Train Loss: 3.7719, Train Steps/Sec: 1.10 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-03-24 15:09:16] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:56<00:00, 52.95s/it] +[2025-03-24 15:18:21] Finish Eval in 20000 steps...███████████████████████████████████████████████████████| 9/9 [07:56<00:00, 52.60s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-03-24 15:18:40] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0020000.pt +[2025-03-24 15:18:42] Removed old checkpoint: checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0016000.pt + 45%|█████████████████████████████████████████████████████████▌ | 20024/44547 [6:50:23<6:25:40, 1.06it/s][2025-03-24 15:19:05] (step=0020025) Train Loss: 3.8250, Train Steps/Sec: 0.04 + 45%|█████████████████████████████████████████████████████████▌ | 20049/44547 [6:50:45<6:11:53, 1.10it/s][2025-03-24 15:19:28] (step=0020050) Train Loss: 3.8167, Train Steps/Sec: 1.10 + 45%|█████████████████████████████████████████████████████████▋ | 20074/44547 [6:51:08<6:09:16, 1.10it/s][2025-03-24 15:19:51] (step=0020075) Train Loss: 3.8311, Train Steps/Sec: 1.10 + 45%|█████████████████████████████████████████████████████████▊ | 20099/44547 [6:51:31<6:10:43, 1.10it/s][2025-03-24 15:20:13] (step=0020100) Train Loss: 3.7652, Train Steps/Sec: 1.10 + 45%|█████████████████████████████████████████████████████████▊ | 20124/44547 [6:51:53<6:11:22, 1.10it/s][2025-03-24 15:20:36] (step=0020125) Train Loss: 3.7086, Train Steps/Sec: 1.10 + 45%|█████████████████████████████████████████████████████████▉ | 20149/44547 [6:52:17<6:10:23, 1.10it/s][2025-03-24 15:21:00] (step=0020150) Train Loss: 3.7857, Train Steps/Sec: 1.06 + 45%|█████████████████████████████████████████████████████████▉ | 20174/44547 [6:52:40<6:08:44, 1.10it/s][2025-03-24 15:21:22] (step=0020175) Train Loss: 3.7811, Train Steps/Sec: 1.10 + 45%|██████████████████████████████████████████████████████████ | 20199/44547 [6:53:03<6:09:51, 1.10it/s][2025-03-24 15:21:45] (step=0020200) Train Loss: 3.9040, Train Steps/Sec: 1.10 + 45%|██████████████████████████████████████████████████████████ | 20224/44547 [6:53:25<6:08:37, 1.10it/s][2025-03-24 15:22:08] (step=0020225) Train Loss: 3.8397, Train Steps/Sec: 1.10 + 45%|██████████████████████████████████████████████████████████▏ | 20249/44547 [6:53:48<6:09:12, 1.10it/s][2025-03-24 15:22:31] (step=0020250) Train Loss: 3.6060, Train Steps/Sec: 1.10 + 46%|██████████████████████████████████████████████████████████▎ | 20274/44547 [6:54:11<6:09:25, 1.10it/s][2025-03-24 15:22:54] (step=0020275) Train Loss: 3.7767, Train Steps/Sec: 1.10 + 46%|██████████████████████████████████████████████████████████▎ | 20299/44547 [6:54:34<6:07:36, 1.10it/s][2025-03-24 15:23:16] (step=0020300) Train Loss: 3.8717, Train Steps/Sec: 1.10 + 46%|██████████████████████████████████████████████████████████▍ | 20324/44547 [6:54:56<6:10:44, 1.09it/s][2025-03-24 15:23:39] (step=0020325) Train Loss: 3.6832, Train Steps/Sec: 1.10 + 46%|██████████████████████████████████████████████████████████▍ | 20349/44547 [6:55:19<6:05:16, 1.10it/s][2025-03-24 15:24:02] (step=0020350) Train Loss: 3.8702, Train Steps/Sec: 1.10 + 46%|██████████████████████████████████████████████████████████▌ | 20374/44547 [6:55:42<6:06:50, 1.10it/s][2025-03-24 15:24:25] (step=0020375) Train Loss: 3.9500, Train Steps/Sec: 1.10 + 46%|██████████████████████████████████████████████████████████▌ | 20399/44547 [6:56:06<6:27:04, 1.04it/s][2025-03-24 15:24:48] (step=0020400) Train Loss: 3.8108, Train Steps/Sec: 1.05 + 46%|██████████████████████████████████████████████████████████▋ | 20424/44547 [6:56:28<6:05:36, 1.10it/s][2025-03-24 15:25:11] (step=0020425) Train Loss: 3.7464, Train Steps/Sec: 1.10 + 46%|██████████████████████████████████████████████████████████▊ | 20449/44547 [6:56:51<6:04:59, 1.10it/s][2025-03-24 15:25:34] (step=0020450) Train Loss: 3.8610, Train Steps/Sec: 1.10 + 46%|██████████████████████████████████████████████████████████▊ | 20474/44547 [6:57:16<8:18:30, 1.24s/it][2025-03-24 15:25:59] (step=0020475) Train Loss: 3.7623, Train Steps/Sec: 1.00 + 46%|██████████████████████████████████████████████████████████▉ | 20499/44547 [6:57:40<6:03:26, 1.10it/s][2025-03-24 15:26:23] (step=0020500) Train Loss: 3.8346, Train Steps/Sec: 1.05 + 46%|██████████████████████████████████████████████████████████▉ | 20524/44547 [6:58:03<6:02:20, 1.10it/s][2025-03-24 15:26:46] (step=0020525) Train Loss: 3.9061, Train Steps/Sec: 1.07 + 46%|███████████████████████████████████████████████████████████ | 20549/44547 [6:58:26<6:02:09, 1.10it/s][2025-03-24 15:27:09] (step=0020550) Train Loss: 3.7506, Train Steps/Sec: 1.10 + 46%|███████████████████████████████████████████████████████████ | 20574/44547 [6:58:51<7:43:14, 1.16s/it][2025-03-24 15:27:33] (step=0020575) Train Loss: 3.7661, Train Steps/Sec: 1.02 + 46%|███████████████████████████████████████████████████████████▏ | 20599/44547 [6:59:13<6:02:20, 1.10it/s][2025-03-24 15:27:56] (step=0020600) Train Loss: 3.8146, Train Steps/Sec: 1.10 + 46%|███████████████████████████████████████████████████████████▎ | 20624/44547 [6:59:36<6:02:52, 1.10it/s][2025-03-24 15:28:19] (step=0020625) Train Loss: 3.8336, Train Steps/Sec: 1.10 + 46%|███████████████████████████████████████████████████████████▎ | 20649/44547 [6:59:59<6:00:04, 1.11it/s][2025-03-24 15:28:41] (step=0020650) Train Loss: 3.9079, Train Steps/Sec: 1.10 + 46%|███████████████████████████████████████████████████████████▍ | 20674/44547 [7:00:21<6:03:13, 1.10it/s][2025-03-24 15:29:04] (step=0020675) Train Loss: 3.7989, Train Steps/Sec: 1.10 + 46%|███████████████████████████████████████████████████████████▍ | 20699/44547 [7:00:44<6:00:54, 1.10it/s][2025-03-24 15:29:27] (step=0020700) Train Loss: 3.7889, Train Steps/Sec: 1.10 + 47%|███████████████████████████████████████████████████████████▌ | 20724/44547 [7:01:07<6:00:07, 1.10it/s][2025-03-24 15:29:50] (step=0020725) Train Loss: 3.8150, Train Steps/Sec: 1.10 + 47%|███████████████████████████████████████████████████████████▌ | 20749/44547 [7:01:30<5:59:51, 1.10it/s][2025-03-24 15:30:12] (step=0020750) Train Loss: 3.8964, Train Steps/Sec: 1.10 + 47%|███████████████████████████████████████████████████████████▋ | 20774/44547 [7:01:52<6:00:16, 1.10it/s][2025-03-24 15:30:35] (step=0020775) Train Loss: 3.7896, Train Steps/Sec: 1.10 + 47%|███████████████████████████████████████████████████████████▊ | 20799/44547 [7:02:15<6:02:29, 1.09it/s][2025-03-24 15:30:58] (step=0020800) Train Loss: 3.8659, Train Steps/Sec: 1.10 + 47%|███████████████████████████████████████████████████████████▊ | 20824/44547 [7:02:39<6:01:53, 1.09it/s][2025-03-24 15:31:22] (step=0020825) Train Loss: 3.8461, Train Steps/Sec: 1.05 + 47%|███████████████████████████████████████████████████████████▉ | 20849/44547 [7:03:02<5:58:26, 1.10it/s][2025-03-24 15:31:44] (step=0020850) Train Loss: 3.9097, Train Steps/Sec: 1.10 + 47%|███████████████████████████████████████████████████████████▉ | 20874/44547 [7:03:24<5:57:18, 1.10it/s][2025-03-24 15:32:07] (step=0020875) Train Loss: 3.7809, Train Steps/Sec: 1.10 + 47%|████████████████████████████████████████████████████████████ | 20899/44547 [7:03:48<6:12:24, 1.06it/s][2025-03-24 15:32:30] (step=0020900) Train Loss: 3.8699, Train Steps/Sec: 1.07 + 47%|████████████████████████████████████████████████████████████ | 20924/44547 [7:04:12<8:05:16, 1.23s/it][2025-03-24 15:32:54] (step=0020925) Train Loss: 3.8049, Train Steps/Sec: 1.05 + 47%|████████████████████████████████████████████████████████████▏ | 20949/44547 [7:04:34<5:59:13, 1.09it/s][2025-03-24 15:33:17] (step=0020950) Train Loss: 3.8038, Train Steps/Sec: 1.10 + 47%|████████████████████████████████████████████████████████████▎ | 20974/44547 [7:04:57<5:56:39, 1.10it/s][2025-03-24 15:33:40] (step=0020975) Train Loss: 3.7395, Train Steps/Sec: 1.10 + 47%|████████████████████████████████████████████████████████████▎ | 20999/44547 [7:05:20<5:56:28, 1.10it/s][2025-03-24 15:34:03] (step=0021000) Train Loss: 3.6907, Train Steps/Sec: 1.10 + 47%|████████████████████████████████████████████████████████████▍ | 21024/44547 [7:05:43<5:56:23, 1.10it/s][2025-03-24 15:34:25] (step=0021025) Train Loss: 3.8140, Train Steps/Sec: 1.10 + 47%|████████████████████████████████████████████████████████████▍ | 21049/44547 [7:06:06<5:54:49, 1.10it/s][2025-03-24 15:34:49] (step=0021050) Train Loss: 3.8399, Train Steps/Sec: 1.06 + 47%|████████████████████████████████████████████████████████████▌ | 21074/44547 [7:06:29<5:54:56, 1.10it/s][2025-03-24 15:35:12] (step=0021075) Train Loss: 3.7291, Train Steps/Sec: 1.10 + 47%|████████████████████████████████████████████████████████████▋ | 21099/44547 [7:06:52<5:54:35, 1.10it/s][2025-03-24 15:35:34] (step=0021100) Train Loss: 3.8885, Train Steps/Sec: 1.10 + 47%|████████████████████████████████████████████████████████████▋ | 21124/44547 [7:07:14<5:54:05, 1.10it/s][2025-03-24 15:35:57] (step=0021125) Train Loss: 3.8098, Train Steps/Sec: 1.10 + 47%|████████████████████████████████████████████████████████████▊ | 21149/44547 [7:07:39<6:26:34, 1.01it/s][2025-03-24 15:36:22] (step=0021150) Train Loss: 3.8700, Train Steps/Sec: 1.00 + 48%|████████████████████████████████████████████████████████████▊ | 21174/44547 [7:08:02<5:52:58, 1.10it/s][2025-03-24 15:36:45] (step=0021175) Train Loss: 3.8682, Train Steps/Sec: 1.10 + 48%|████████████████████████████████████████████████████████████▉ | 21199/44547 [7:08:26<6:33:32, 1.01s/it][2025-03-24 15:37:08] (step=0021200) Train Loss: 3.6633, Train Steps/Sec: 1.05 + 48%|████████████████████████████████████████████████████████████▉ | 21224/44547 [7:08:48<5:53:05, 1.10it/s][2025-03-24 15:37:31] (step=0021225) Train Loss: 3.8380, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████ | 21249/44547 [7:09:11<5:53:51, 1.10it/s][2025-03-24 15:37:54] (step=0021250) Train Loss: 3.7485, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████▏ | 21274/44547 [7:09:34<5:53:55, 1.10it/s][2025-03-24 15:38:17] (step=0021275) Train Loss: 3.9122, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████▏ | 21299/44547 [7:09:57<5:53:16, 1.10it/s][2025-03-24 15:38:39] (step=0021300) Train Loss: 3.8057, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████▎ | 21324/44547 [7:10:19<5:51:09, 1.10it/s][2025-03-24 15:39:02] (step=0021325) Train Loss: 3.7669, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████▎ | 21349/44547 [7:10:42<5:52:47, 1.10it/s][2025-03-24 15:39:25] (step=0021350) Train Loss: 3.8626, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████▍ | 21374/44547 [7:11:05<5:51:52, 1.10it/s][2025-03-24 15:39:48] (step=0021375) Train Loss: 3.8623, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████▍ | 21399/44547 [7:11:28<5:51:08, 1.10it/s][2025-03-24 15:40:10] (step=0021400) Train Loss: 3.8982, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████▌ | 21424/44547 [7:11:51<6:29:56, 1.01s/it][2025-03-24 15:40:34] (step=0021425) Train Loss: 3.6369, Train Steps/Sec: 1.06 + 48%|█████████████████████████████████████████████████████████████▋ | 21449/44547 [7:12:14<5:49:12, 1.10it/s][2025-03-24 15:40:57] (step=0021450) Train Loss: 3.9659, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████▋ | 21474/44547 [7:12:37<5:50:09, 1.10it/s][2025-03-24 15:41:19] (step=0021475) Train Loss: 3.6314, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████▊ | 21499/44547 [7:12:59<5:47:54, 1.10it/s][2025-03-24 15:41:42] (step=0021500) Train Loss: 3.7069, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████▊ | 21524/44547 [7:13:22<5:47:51, 1.10it/s][2025-03-24 15:42:05] (step=0021525) Train Loss: 3.8385, Train Steps/Sec: 1.10 + 48%|█████████████████████████████████████████████████████████████▉ | 21549/44547 [7:13:47<6:11:24, 1.03it/s][2025-03-24 15:42:30] (step=0021550) Train Loss: 3.7781, Train Steps/Sec: 1.00 + 48%|█████████████████████████████████████████████████████████████▉ | 21574/44547 [7:14:10<6:26:30, 1.01s/it][2025-03-24 15:42:53] (step=0021575) Train Loss: 3.7920, Train Steps/Sec: 1.07 + 48%|██████████████████████████████████████████████████████████████ | 21599/44547 [7:14:33<5:46:50, 1.10it/s][2025-03-24 15:43:16] (step=0021600) Train Loss: 3.7023, Train Steps/Sec: 1.10 + 49%|██████████████████████████████████████████████████████████████▏ | 21624/44547 [7:14:57<5:47:31, 1.10it/s][2025-03-24 15:43:39] (step=0021625) Train Loss: 3.8334, Train Steps/Sec: 1.06 + 49%|██████████████████████████████████████████████████████████████▏ | 21649/44547 [7:15:20<5:47:48, 1.10it/s][2025-03-24 15:44:02] (step=0021650) Train Loss: 3.9187, Train Steps/Sec: 1.10 + 49%|██████████████████████████████████████████████████████████████▎ | 21674/44547 [7:15:42<5:45:41, 1.10it/s][2025-03-24 15:44:25] (step=0021675) Train Loss: 3.7860, Train Steps/Sec: 1.10 + 49%|██████████████████████████████████████████████████████████████▎ | 21699/44547 [7:16:05<5:46:45, 1.10it/s][2025-03-24 15:44:48] (step=0021700) Train Loss: 3.9311, Train Steps/Sec: 1.10 + 49%|██████████████████████████████████████████████████████████████▍ | 21724/44547 [7:16:28<5:44:24, 1.10it/s][2025-03-24 15:45:10] (step=0021725) Train Loss: 3.7109, Train Steps/Sec: 1.10 + 49%|██████████████████████████████████████████████████████████████▍ | 21749/44547 [7:16:50<5:44:18, 1.10it/s][2025-03-24 15:45:33] (step=0021750) Train Loss: 3.9228, Train Steps/Sec: 1.10 + 49%|██████████████████████████████████████████████████████████████▌ | 21774/44547 [7:17:13<5:44:40, 1.10it/s][2025-03-24 15:45:56] (step=0021775) Train Loss: 3.8376, Train Steps/Sec: 1.10 + 49%|██████████████████████████████████████████████████████████████▋ | 21799/44547 [7:17:37<5:44:01, 1.10it/s][2025-03-24 15:46:19] (step=0021800) Train Loss: 3.8201, Train Steps/Sec: 1.06 + 49%|██████████████████████████████████████████████████████████████▋ | 21824/44547 [7:18:00<5:43:06, 1.10it/s][2025-03-24 15:46:42] (step=0021825) Train Loss: 3.8652, Train Steps/Sec: 1.10 + 49%|██████████████████████████████████████████████████████████████▊ | 21849/44547 [7:18:23<6:12:52, 1.01it/s][2025-03-24 15:47:06] (step=0021850) Train Loss: 3.9279, Train Steps/Sec: 1.05 + 49%|██████████████████████████████████████████████████████████████▊ | 21874/44547 [7:18:46<5:45:15, 1.09it/s][2025-03-24 15:47:29] (step=0021875) Train Loss: 3.7959, Train Steps/Sec: 1.10 + 49%|██████████████████████████████████████████████████████████████▉ | 21899/44547 [7:19:10<5:42:35, 1.10it/s][2025-03-24 15:47:54] (step=0021900) Train Loss: 3.7760, Train Steps/Sec: 1.05 + 49%|██████████████████████████████████████████████████████████████▉ | 21924/44547 [7:19:34<5:44:21, 1.09it/s][2025-03-24 15:48:16] (step=0021925) Train Loss: 3.7868, Train Steps/Sec: 1.10 + 49%|███████████████████████████████████████████████████████████████ | 21949/44547 [7:19:57<5:44:01, 1.09it/s][2025-03-24 15:48:39] (step=0021950) Train Loss: 3.7188, Train Steps/Sec: 1.10 + 49%|███████████████████████████████████████████████████████████████▏ | 21974/44547 [7:20:19<5:40:49, 1.10it/s][2025-03-24 15:49:02] (step=0021975) Train Loss: 3.9292, Train Steps/Sec: 1.10 + 49%|███████████████████████████████████████████████████████████████▏ | 21999/44547 [7:20:42<5:40:53, 1.10it/s][2025-03-24 15:49:25] (step=0022000) Train Loss: 3.7860, Train Steps/Sec: 1.10 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-03-24 15:49:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:57<00:00, 53.10s/it] +[2025-03-24 15:58:30] Finish Eval in 22000 steps...███████████████████████████████████████████████████████| 9/9 [07:57<00:00, 52.74s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... + 49%|███████████████████████████████████████████████████████████████▎ | 22024/44547 [7:30:11<5:53:10, 1.06it/s][2025-03-24 15:58:54] (step=0022025) Train Loss: 3.7337, Train Steps/Sec: 0.04 + 49%|███████████████████████████████████████████████████████████████▎ | 22049/44547 [7:30:34<5:40:20, 1.10it/s][2025-03-24 15:59:16] (step=0022050) Train Loss: 3.7349, Train Steps/Sec: 1.10 + 50%|███████████████████████████████████████████████████████████████▍ | 22074/44547 [7:30:57<5:44:08, 1.09it/s][2025-03-24 15:59:40] (step=0022075) Train Loss: 3.8537, Train Steps/Sec: 1.07 + 50%|███████████████████████████████████████████████████████████████▍ | 22099/44547 [7:31:20<5:41:36, 1.10it/s][2025-03-24 16:00:03] (step=0022100) Train Loss: 3.7463, Train Steps/Sec: 1.10 + 50%|███████████████████████████████████████████████████████████████▌ | 22124/44547 [7:31:44<5:41:18, 1.09it/s][2025-03-24 16:00:26] (step=0022125) Train Loss: 3.7548, Train Steps/Sec: 1.05 + 50%|███████████████████████████████████████████████████████████████▋ | 22149/44547 [7:32:06<5:37:57, 1.10it/s][2025-03-24 16:00:49] (step=0022150) Train Loss: 3.6521, Train Steps/Sec: 1.10 + 50%|███████████████████████████████████████████████████████████████▋ | 22174/44547 [7:32:29<5:37:58, 1.10it/s][2025-03-24 16:01:12] (step=0022175) Train Loss: 3.8797, Train Steps/Sec: 1.10 + 50%|███████████████████████████████████████████████████████████████▊ | 22199/44547 [7:32:53<5:53:05, 1.05it/s][2025-03-24 16:01:36] (step=0022200) Train Loss: 3.8241, Train Steps/Sec: 1.05 + 50%|███████████████████████████████████████████████████████████████▊ | 22224/44547 [7:33:16<5:38:07, 1.10it/s][2025-03-24 16:01:58] (step=0022225) Train Loss: 3.7901, Train Steps/Sec: 1.10 + 50%|███████████████████████████████████████████████████████████████▉ | 22249/44547 [7:33:38<5:35:47, 1.11it/s][2025-03-24 16:02:21] (step=0022250) Train Loss: 3.7686, Train Steps/Sec: 1.10 + 50%|████████████████████████████████████████████████████████████████ | 22274/44547 [7:34:01<5:37:04, 1.10it/s][2025-03-24 16:02:44] (step=0022275) Train Loss: 3.9818, Train Steps/Sec: 1.10 + 50%|████████████████████████████████████████████████████████████████ | 22299/44547 [7:34:24<5:36:56, 1.10it/s][2025-03-24 16:03:06] (step=0022300) Train Loss: 3.7481, Train Steps/Sec: 1.10 + 50%|████████████████████████████████████████████████████████████████▏ | 22324/44547 [7:34:46<5:35:31, 1.10it/s][2025-03-24 16:03:29] (step=0022325) Train Loss: 3.8330, Train Steps/Sec: 1.10 + 50%|████████████████████████████████████████████████████████████████▏ | 22349/44547 [7:35:09<5:35:44, 1.10it/s][2025-03-24 16:03:52] (step=0022350) Train Loss: 3.8599, Train Steps/Sec: 1.10 + 50%|████████████████████████████████████████████████████████████████▎ | 22374/44547 [7:35:33<7:26:41, 1.21s/it][2025-03-24 16:04:16] (step=0022375) Train Loss: 3.8269, Train Steps/Sec: 1.05 + 50%|████████████████████████████████████████████████████████████████▎ | 22399/44547 [7:35:56<5:36:20, 1.10it/s][2025-03-24 16:04:38] (step=0022400) Train Loss: 3.6981, Train Steps/Sec: 1.10 + 50%|████████████████████████████████████████████████████████████████▍ | 22424/44547 [7:36:19<5:35:41, 1.10it/s][2025-03-24 16:05:02] (step=0022425) Train Loss: 3.6829, Train Steps/Sec: 1.06 + 50%|████████████████████████████████████████████████████████████████▌ | 22449/44547 [7:36:42<5:43:17, 1.07it/s][2025-03-24 16:05:25] (step=0022450) Train Loss: 4.0023, Train Steps/Sec: 1.09 + 50%|████████████████████████████████████████████████████████████████▌ | 22474/44547 [7:37:05<5:34:57, 1.10it/s][2025-03-24 16:05:47] (step=0022475) Train Loss: 3.7457, Train Steps/Sec: 1.10 + 51%|████████████████████████████████████████████████████████████████▋ | 22499/44547 [7:37:28<5:35:14, 1.10it/s][2025-03-24 16:06:10] (step=0022500) Train Loss: 3.8221, Train Steps/Sec: 1.10 + 51%|████████████████████████████████████████████████████████████████▋ | 22524/44547 [7:37:50<5:34:35, 1.10it/s][2025-03-24 16:06:33] (step=0022525) Train Loss: 3.7872, Train Steps/Sec: 1.10 + 51%|████████████████████████████████████████████████████████████████▊ | 22549/44547 [7:38:13<5:33:16, 1.10it/s][2025-03-24 16:06:56] (step=0022550) Train Loss: 3.8212, Train Steps/Sec: 1.10 + 51%|████████████████████████████████████████████████████████████████▊ | 22574/44547 [7:38:36<5:34:28, 1.09it/s][2025-03-24 16:07:19] (step=0022575) Train Loss: 3.8870, Train Steps/Sec: 1.07 + 51%|████████████████████████████████████████████████████████████████▉ | 22599/44547 [7:39:00<5:42:19, 1.07it/s][2025-03-24 16:07:43] (step=0022600) Train Loss: 3.7920, Train Steps/Sec: 1.05 + 51%|█████████████████████████████████████████████████████████████████ | 22624/44547 [7:39:23<5:33:10, 1.10it/s][2025-03-24 16:08:06] (step=0022625) Train Loss: 3.6706, Train Steps/Sec: 1.10 + 51%|█████████████████████████████████████████████████████████████████ | 22649/44547 [7:39:47<6:09:03, 1.01s/it][2025-03-24 16:08:31] (step=0022650) Train Loss: 3.7876, Train Steps/Sec: 1.00 + 51%|█████████████████████████████████████████████████████████████████▏ | 22674/44547 [7:40:11<5:31:58, 1.10it/s][2025-03-24 16:08:53] (step=0022675) Train Loss: 3.6984, Train Steps/Sec: 1.10 + 51%|█████████████████████████████████████████████████████████████████▏ | 22699/44547 [7:40:34<5:32:44, 1.09it/s][2025-03-24 16:09:17] (step=0022700) Train Loss: 3.7029, Train Steps/Sec: 1.05 + 51%|█████████████████████████████████████████████████████████████████▎ | 22724/44547 [7:40:57<5:30:24, 1.10it/s][2025-03-24 16:09:40] (step=0022725) Train Loss: 3.7628, Train Steps/Sec: 1.10 + 51%|█████████████████████████████████████████████████████████████████▎ | 22749/44547 [7:41:20<5:29:31, 1.10it/s][2025-03-24 16:10:02] (step=0022750) Train Loss: 3.7639, Train Steps/Sec: 1.10 + 51%|█████████████████████████████████████████████████████████████████▍ | 22774/44547 [7:41:42<5:30:30, 1.10it/s][2025-03-24 16:10:25] (step=0022775) Train Loss: 3.8560, Train Steps/Sec: 1.10 + 51%|█████████████████████████████████████████████████████████████████▌ | 22799/44547 [7:42:05<5:29:33, 1.10it/s][2025-03-24 16:10:48] (step=0022800) Train Loss: 3.8734, Train Steps/Sec: 1.10 + 51%|█████████████████████████████████████████████████████████████████▌ | 22824/44547 [7:42:28<5:28:23, 1.10it/s][2025-03-24 16:11:11] (step=0022825) Train Loss: 3.6483, Train Steps/Sec: 1.10 + 51%|█████████████████████████████████████████████████████████████████▋ | 22849/44547 [7:42:52<7:20:28, 1.22s/it][2025-03-24 16:11:34] (step=0022850) Train Loss: 3.7839, Train Steps/Sec: 1.05 + 51%|█████████████████████████████████████████████████████████████████▋ | 22874/44547 [7:43:14<5:27:05, 1.10it/s][2025-03-24 16:11:57] (step=0022875) Train Loss: 3.7585, Train Steps/Sec: 1.10 + 51%|█████████████████████████████████████████████████████████████████▊ | 22899/44547 [7:43:37<5:28:45, 1.10it/s][2025-03-24 16:12:20] (step=0022900) Train Loss: 3.7584, Train Steps/Sec: 1.10 + 51%|█████████████████████████████████████████████████████████████████▊ | 22924/44547 [7:44:01<5:36:27, 1.07it/s][2025-03-24 16:12:43] (step=0022925) Train Loss: 3.8295, Train Steps/Sec: 1.05 + 52%|█████████████████████████████████████████████████████████████████▉ | 22949/44547 [7:44:23<5:26:14, 1.10it/s][2025-03-24 16:13:06] (step=0022950) Train Loss: 3.7967, Train Steps/Sec: 1.10 + 52%|██████████████████████████████████████████████████████████████████ | 22974/44547 [7:44:47<5:49:18, 1.03it/s][2025-03-24 16:13:30] (step=0022975) Train Loss: 3.7504, Train Steps/Sec: 1.06 + 52%|██████████████████████████████████████████████████████████████████ | 22999/44547 [7:45:10<5:25:52, 1.10it/s][2025-03-24 16:13:52] (step=0023000) Train Loss: 3.8730, Train Steps/Sec: 1.10 + 52%|██████████████████████████████████████████████████████████████████▏ | 23024/44547 [7:45:33<5:25:38, 1.10it/s][2025-03-24 16:14:15] (step=0023025) Train Loss: 3.7314, Train Steps/Sec: 1.10 + 52%|██████████████████████████████████████████████████████████████████▏ | 23049/44547 [7:45:55<5:26:11, 1.10it/s][2025-03-24 16:14:38] (step=0023050) Train Loss: 3.6846, Train Steps/Sec: 1.10 + 52%|██████████████████████████████████████████████████████████████████▎ | 23074/44547 [7:46:18<5:24:25, 1.10it/s][2025-03-24 16:15:01] (step=0023075) Train Loss: 3.8173, Train Steps/Sec: 1.10 + 52%|██████████████████████████████████████████████████████████████████▎ | 23099/44547 [7:46:42<5:56:23, 1.00it/s][2025-03-24 16:15:25] (step=0023100) Train Loss: 3.7309, Train Steps/Sec: 1.05 + 52%|██████████████████████████████████████████████████████████████████▍ | 23124/44547 [7:47:05<5:24:29, 1.10it/s][2025-03-24 16:15:47] (step=0023125) Train Loss: 3.8529, Train Steps/Sec: 1.10 + 52%|██████████████████████████████████████████████████████████████████▌ | 23149/44547 [7:47:27<5:26:02, 1.09it/s][2025-03-24 16:16:10] (step=0023150) Train Loss: 3.8705, Train Steps/Sec: 1.10 + 52%|██████████████████████████████████████████████████████████████████▌ | 23174/44547 [7:47:50<5:23:17, 1.10it/s][2025-03-24 16:16:33] (step=0023175) Train Loss: 3.7021, Train Steps/Sec: 1.10 + 52%|██████████████████████████████████████████████████████████████████▋ | 23199/44547 [7:48:13<5:23:46, 1.10it/s][2025-03-24 16:16:56] (step=0023200) Train Loss: 3.7131, Train Steps/Sec: 1.10 + 52%|██████████████████████████████████████████████████████████████████▋ | 23224/44547 [7:48:36<5:22:12, 1.10it/s][2025-03-24 16:17:18] (step=0023225) Train Loss: 3.8292, Train Steps/Sec: 1.10 + 52%|██████████████████████████████████████████████████████████████████▊ | 23249/44547 [7:48:59<5:22:52, 1.10it/s][2025-03-24 16:17:42] (step=0023250) Train Loss: 3.8947, Train Steps/Sec: 1.07 + 52%|██████████████████████████████████████████████████████████████████▊ | 23274/44547 [7:49:22<5:23:01, 1.10it/s][2025-03-24 16:18:04] (step=0023275) Train Loss: 3.7813, Train Steps/Sec: 1.10 + 52%|██████████████████████████████████████████████████████████████████▉ | 23299/44547 [7:49:45<5:33:34, 1.06it/s][2025-03-24 16:18:28] (step=0023300) Train Loss: 3.7857, Train Steps/Sec: 1.06 + 52%|███████████████████████████████████████████████████████████████████ | 23324/44547 [7:50:09<6:28:26, 1.10s/it][2025-03-24 16:18:52] (step=0023325) Train Loss: 3.8995, Train Steps/Sec: 1.06 + 52%|███████████████████████████████████████████████████████████████████ | 23349/44547 [7:50:32<5:20:31, 1.10it/s][2025-03-24 16:19:14] (step=0023350) Train Loss: 3.7723, Train Steps/Sec: 1.10 + 52%|███████████████████████████████████████████████████████████████████▏ | 23374/44547 [7:50:55<5:20:06, 1.10it/s][2025-03-24 16:19:37] (step=0023375) Train Loss: 3.7576, Train Steps/Sec: 1.10 + 53%|███████████████████████████████████████████████████████████████████▏ | 23399/44547 [7:51:17<5:20:02, 1.10it/s][2025-03-24 16:20:00] (step=0023400) Train Loss: 3.7430, Train Steps/Sec: 1.10 + 53%|███████████████████████████████████████████████████████████████████▎ | 23424/44547 [7:51:41<5:22:17, 1.09it/s][2025-03-24 16:20:24] (step=0023425) Train Loss: 3.9693, Train Steps/Sec: 1.04 + 53%|███████████████████████████████████████████████████████████████████▍ | 23449/44547 [7:52:04<5:18:06, 1.11it/s][2025-03-24 16:20:47] (step=0023450) Train Loss: 3.8228, Train Steps/Sec: 1.10 + 53%|███████████████████████████████████████████████████████████████████▍ | 23474/44547 [7:52:27<5:17:40, 1.11it/s][2025-03-24 16:21:09] (step=0023475) Train Loss: 3.8947, Train Steps/Sec: 1.10 + 53%|███████████████████████████████████████████████████████████████████▌ | 23499/44547 [7:52:49<5:19:37, 1.10it/s][2025-03-24 16:21:32] (step=0023500) Train Loss: 3.8286, Train Steps/Sec: 1.10 + 53%|███████████████████████████████████████████████████████████████████▌ | 23524/44547 [7:53:12<5:19:41, 1.10it/s][2025-03-24 16:21:55] (step=0023525) Train Loss: 3.8494, Train Steps/Sec: 1.10 + 53%|███████████████████████████████████████████████████████████████████▋ | 23549/44547 [7:53:35<5:17:23, 1.10it/s][2025-03-24 16:22:17] (step=0023550) Train Loss: 3.7075, Train Steps/Sec: 1.10 + 53%|███████████████████████████████████████████████████████████████████▋ | 23574/44547 [7:53:57<5:17:28, 1.10it/s][2025-03-24 16:22:40] (step=0023575) Train Loss: 3.7215, Train Steps/Sec: 1.10 + 53%|███████████████████████████████████████████████████████████████████▊ | 23599/44547 [7:54:20<5:16:56, 1.10it/s][2025-03-24 16:23:03] (step=0023600) Train Loss: 3.7512, Train Steps/Sec: 1.10 + 53%|███████████████████████████████████████████████████████████████████▉ | 23624/44547 [7:54:43<5:15:57, 1.10it/s][2025-03-24 16:23:25] (step=0023625) Train Loss: 3.6911, Train Steps/Sec: 1.10 + 53%|███████████████████████████████████████████████████████████████████▉ | 23649/44547 [7:55:06<5:17:14, 1.10it/s][2025-03-24 16:23:48] (step=0023650) Train Loss: 3.6616, Train Steps/Sec: 1.10 + 53%|████████████████████████████████████████████████████████████████████ | 23674/44547 [7:55:28<5:16:56, 1.10it/s][2025-03-24 16:24:11] (step=0023675) Train Loss: 3.7406, Train Steps/Sec: 1.10 + 53%|████████████████████████████████████████████████████████████████████ | 23699/44547 [7:55:52<5:17:49, 1.09it/s][2025-03-24 16:24:35] (step=0023700) Train Loss: 3.7819, Train Steps/Sec: 1.05 + 53%|████████████████████████████████████████████████████████████████████▏ | 23724/44547 [7:56:16<5:16:07, 1.10it/s][2025-03-24 16:24:58] (step=0023725) Train Loss: 3.8042, Train Steps/Sec: 1.05 + 53%|████████████████████████████████████████████████████████████████████▏ | 23749/44547 [7:56:39<5:14:55, 1.10it/s][2025-03-24 16:25:22] (step=0023750) Train Loss: 3.8237, Train Steps/Sec: 1.06 + 53%|████████████████████████████████████████████████████████████████████▎ | 23774/44547 [7:57:03<5:32:23, 1.04it/s][2025-03-24 16:25:46] (step=0023775) Train Loss: 3.8001, Train Steps/Sec: 1.05 + 53%|████████████████████████████████████████████████████████████████████▍ | 23799/44547 [7:57:26<5:14:28, 1.10it/s][2025-03-24 16:26:09] (step=0023800) Train Loss: 3.9019, Train Steps/Sec: 1.10 + 53%|████████████████████████████████████████████████████████████████████▍ | 23824/44547 [7:57:49<5:14:22, 1.10it/s][2025-03-24 16:26:31] (step=0023825) Train Loss: 3.7654, Train Steps/Sec: 1.10 + 54%|████████████████████████████████████████████████████████████████████▌ | 23849/44547 [7:58:11<5:14:42, 1.10it/s][2025-03-24 16:26:54] (step=0023850) Train Loss: 3.8013, Train Steps/Sec: 1.10 + 54%|████████████████████████████████████████████████████████████████████▌ | 23874/44547 [7:58:35<5:20:53, 1.07it/s][2025-03-24 16:27:18] (step=0023875) Train Loss: 3.7060, Train Steps/Sec: 1.06 + 54%|████████████████████████████████████████████████████████████████████▋ | 23899/44547 [7:58:58<5:12:10, 1.10it/s][2025-03-24 16:27:41] (step=0023900) Train Loss: 3.7552, Train Steps/Sec: 1.10 + 54%|████████████████████████████████████████████████████████████████████▋ | 23924/44547 [7:59:21<5:47:13, 1.01s/it][2025-03-24 16:28:04] (step=0023925) Train Loss: 3.8278, Train Steps/Sec: 1.07 + 54%|████████████████████████████████████████████████████████████████████▊ | 23949/44547 [7:59:44<5:11:43, 1.10it/s][2025-03-24 16:28:27] (step=0023950) Train Loss: 3.7485, Train Steps/Sec: 1.10 + 54%|████████████████████████████████████████████████████████████████████▉ | 23974/44547 [8:00:08<5:16:39, 1.08it/s][2025-03-24 16:28:50] (step=0023975) Train Loss: 3.7487, Train Steps/Sec: 1.05 + 54%|████████████████████████████████████████████████████████████████████▉ | 23999/44547 [8:00:30<5:10:02, 1.10it/s][2025-03-24 16:29:13] (step=0024000) Train Loss: 3.8438, Train Steps/Sec: 1.10 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-03-24 16:29:13] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [07:56<00:00, 52.90s/it] +[2025-03-24 16:38:16] Finish Eval in 24000 steps...███████████████████████████████████████████████████████| 9/9 [07:55<00:00, 52.73s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-03-24 16:38:34] Saved checkpoint to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0024000.pt +[2025-03-24 16:38:36] Removed old checkpoint: checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/002-GPT-XL/checkpoints/0020000.pt + 54%|█████████████████████████████████████████████████████████████████████ | 24024/44547 [8:10:16<5:22:15, 1.06it/s][2025-03-24 16:38:59] (step=0024025) Train Loss: 3.8032, Train Steps/Sec: 0.04 + 54%|█████████████████████████████████████████████████████████████████████ | 24049/44547 [8:10:39<5:11:55, 1.10it/s][2025-03-24 16:39:22] (step=0024050) Train Loss: 3.7325, Train Steps/Sec: 1.10 + 54%|█████████████████████████████████████████████████████████████████████▏ | 24074/44547 [8:11:02<5:12:13, 1.09it/s][2025-03-24 16:39:45] (step=0024075) Train Loss: 3.7026, Train Steps/Sec: 1.09 + 54%|█████████████████████████████████████████████████████████████████████▏ | 24099/44547 [8:11:25<5:09:00, 1.10it/s][2025-03-24 16:40:07] (step=0024100) Train Loss: 3.8385, Train Steps/Sec: 1.10 + 54%|█████████████████████████████████████████████████████████████████████▎ | 24124/44547 [8:11:48<5:11:49, 1.09it/s][2025-03-24 16:40:30] (step=0024125) Train Loss: 3.6707, Train Steps/Sec: 1.09 + 54%|█████████████████████████████████████████████████████████████████████▍ | 24149/44547 [8:12:10<5:09:56, 1.10it/s][2025-03-24 16:40:53] (step=0024150) Train Loss: 3.7718, Train Steps/Sec: 1.10 + 54%|█████████████████████████████████████████████████████████████████████▍ | 24174/44547 [8:12:33<5:11:09, 1.09it/s][2025-03-24 16:41:16] (step=0024175) Train Loss: 3.8088, Train Steps/Sec: 1.10 + 54%|█████████████████████████████████████████████████████████████████████▌ | 24199/44547 [8:12:56<5:08:16, 1.10it/s][2025-03-24 16:41:38] (step=0024200) Train Loss: 3.7400, Train Steps/Sec: 1.10 + 54%|█████████████████████████████████████████████████████████████████████▌ | 24224/44547 [8:13:19<5:09:34, 1.09it/s][2025-03-24 16:42:01] (step=0024225) Train Loss: 3.7887, Train Steps/Sec: 1.10 + 54%|█████████████████████████████████████████████████████████████████████▋ | 24249/44547 [8:13:41<5:06:10, 1.10it/s][2025-03-24 16:42:24] (step=0024250) Train Loss: 3.7431, Train Steps/Sec: 1.10 + 54%|█████████████████████████████████████████████████████████████████████▋ | 24274/44547 [8:14:04<5:08:53, 1.09it/s][2025-03-24 16:42:47] (step=0024275) Train Loss: 3.7625, Train Steps/Sec: 1.10 + 55%|█████████████████████████████████████████████████████████████████████▊ | 24299/44547 [8:14:28<5:06:31, 1.10it/s][2025-03-24 16:43:11] (step=0024300) Train Loss: 3.8486, Train Steps/Sec: 1.05 + 55%|█████████████████████████████████████████████████████████████████████▉ | 24324/44547 [8:14:53<5:07:15, 1.10it/s][2025-03-24 16:43:35] (step=0024325) Train Loss: 3.7825, Train Steps/Sec: 1.01 + 55%|█████████████████████████████████████████████████████████████████████▉ | 24349/44547 [8:15:15<5:06:19, 1.10it/s][2025-03-24 16:43:58] (step=0024350) Train Loss: 3.7757, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████ | 24374/44547 [8:15:39<5:06:57, 1.10it/s][2025-03-24 16:44:22] (step=0024375) Train Loss: 3.8332, Train Steps/Sec: 1.05 + 55%|██████████████████████████████████████████████████████████████████████ | 24399/44547 [8:16:02<5:05:57, 1.10it/s][2025-03-24 16:44:45] (step=0024400) Train Loss: 3.6699, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████▏ | 24424/44547 [8:16:25<5:05:06, 1.10it/s][2025-03-24 16:45:07] (step=0024425) Train Loss: 3.7842, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████▎ | 24449/44547 [8:16:47<5:04:12, 1.10it/s][2025-03-24 16:45:30] (step=0024450) Train Loss: 3.7548, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████▎ | 24474/44547 [8:17:10<5:04:27, 1.10it/s][2025-03-24 16:45:53] (step=0024475) Train Loss: 3.8532, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████▍ | 24499/44547 [8:17:33<5:04:08, 1.10it/s][2025-03-24 16:46:16] (step=0024500) Train Loss: 3.7482, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████▍ | 24524/44547 [8:17:56<5:04:16, 1.10it/s][2025-03-24 16:46:39] (step=0024525) Train Loss: 3.7011, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████▌ | 24549/44547 [8:18:19<5:03:34, 1.10it/s][2025-03-24 16:47:01] (step=0024550) Train Loss: 3.8169, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████▌ | 24574/44547 [8:18:43<5:08:14, 1.08it/s][2025-03-24 16:47:26] (step=0024575) Train Loss: 3.7878, Train Steps/Sec: 1.01 + 55%|██████████████████████████████████████████████████████████████████████▋ | 24599/44547 [8:19:06<5:02:00, 1.10it/s][2025-03-24 16:47:49] (step=0024600) Train Loss: 3.7843, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████▊ | 24624/44547 [8:19:29<5:03:56, 1.09it/s][2025-03-24 16:48:12] (step=0024625) Train Loss: 3.8253, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████▊ | 24649/44547 [8:19:52<5:02:03, 1.10it/s][2025-03-24 16:48:34] (step=0024650) Train Loss: 3.7400, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████▉ | 24674/44547 [8:20:15<5:02:18, 1.10it/s][2025-03-24 16:48:57] (step=0024675) Train Loss: 3.8731, Train Steps/Sec: 1.10 + 55%|██████████████████████████████████████████████████████████████████████▉ | 24699/44547 [8:20:37<4:59:55, 1.10it/s][2025-03-24 16:49:20] (step=0024700) Train Loss: 3.7555, Train Steps/Sec: 1.10 + 56%|███████████████████████████████████████████████████████████████████████ | 24724/44547 [8:21:00<5:02:10, 1.09it/s][2025-03-24 16:49:43] (step=0024725) Train Loss: 3.8252, Train Steps/Sec: 1.10 + 56%|███████████████████████████████████████████████████████████████████████ | 24749/44547 [8:21:23<5:00:36, 1.10it/s][2025-03-24 16:50:05] (step=0024750) Train Loss: 3.7180, Train Steps/Sec: 1.10 + 56%|███████████████████████████████████████████████████████████████████████▏ | 24774/44547 [8:21:46<5:00:01, 1.10it/s][2025-03-24 16:50:28] (step=0024775) Train Loss: 3.7159, Train Steps/Sec: 1.10 + 56%|███████████████████████████████████████████████████████████████████████▎ | 24799/44547 [8:22:08<5:00:37, 1.09it/s][2025-03-24 16:50:51] (step=0024800) Train Loss: 3.8233, Train Steps/Sec: 1.10 + 56%|███████████████████████████████████████████████████████████████████████▎ | 24824/44547 [8:22:32<4:59:06, 1.10it/s][2025-03-24 16:51:15] (step=0024825) Train Loss: 3.8312, Train Steps/Sec: 1.05 + 56%|███████████████████████████████████████████████████████████████████████▍ | 24849/44547 [8:22:55<4:58:26, 1.10it/s][2025-03-24 16:51:37] (step=0024850) Train Loss: 3.8430, Train Steps/Sec: 1.10 + 56%|███████████████████████████████████████████████████████████████████████▍ | 24874/44547 [8:23:18<4:57:37, 1.10it/s][2025-03-24 16:52:00] (step=0024875) Train Loss: 3.7718, Train Steps/Sec: 1.10 + 56%|███████████████████████████████████████████████████████████████████████▌ | 24899/44547 [8:23:41<4:58:29, 1.10it/s][2025-03-24 16:52:24] (step=0024900) Train Loss: 3.6848, Train Steps/Sec: 1.05 + 56%|███████████████████████████████████████████████████████████████████████▌ | 24924/44547 [8:24:05<4:57:08, 1.10it/s][2025-03-24 16:52:48] (step=0024925) Train Loss: 3.8005, Train Steps/Sec: 1.05 + 56%|███████████████████████████████████████████████████████████████████████▋ | 24949/44547 [8:24:28<4:57:39, 1.10it/s][2025-03-24 16:53:11] (step=0024950) Train Loss: 3.8165, Train Steps/Sec: 1.10 + 56%|███████████████████████████████████████████████████████████████████████▊ | 24974/44547 [8:24:51<4:57:30, 1.10it/s][2025-03-24 16:53:33] (step=0024975) Train Loss: 3.7732, Train Steps/Sec: 1.10 + 56%|███████████████████████████████████████████████████████████████████████▊ | 24999/44547 [8:25:15<4:57:45, 1.09it/s][2025-03-24 16:53:58] (step=0025000) Train Loss: 3.8033, Train Steps/Sec: 1.03 + 56%|███████████████████████████████████████████████████████████████████████▉ | 25024/44547 [8:25:38<4:56:58, 1.10it/s][2025-03-24 16:54:20] (step=0025025) Train Loss: 3.6945, Train Steps/Sec: 1.10 + 56%|███████████████████████████████████████████████████████████████████████▉ | 25049/44547 [8:26:01<4:56:50, 1.09it/s][2025-03-24 16:54:43] (step=0025050) Train Loss: 3.6173, Train Steps/Sec: 1.10 + 56%|████████████████████████████████████████████████████████████████████████ | 25074/44547 [8:26:23<4:55:29, 1.10it/s][2025-03-24 16:55:06] (step=0025075) Train Loss: 3.7128, Train Steps/Sec: 1.10 + 56%|████████████████████████████████████████████████████████████████████████ | 25099/44547 [8:26:47<4:57:48, 1.09it/s][2025-03-24 16:55:30] (step=0025100) Train Loss: 3.7611, Train Steps/Sec: 1.05 + 56%|████████████████████████████████████████████████████████████████████████▏ | 25124/44547 [8:27:10<4:55:37, 1.09it/s][2025-03-24 16:55:53] (step=0025125) Train Loss: 3.7503, Train Steps/Sec: 1.09 + 56%|████████████████████████████████████████████████████████████████████████▎ | 25149/44547 [8:27:33<4:53:55, 1.10it/s][2025-03-24 16:56:16] (step=0025150) Train Loss: 3.7286, Train Steps/Sec: 1.10 + 57%|████████████████████████████████████████████████████████████████████████▎ | 25174/44547 [8:27:57<4:52:22, 1.10it/s][2025-03-24 16:56:39] (step=0025175) Train Loss: 3.6947, Train Steps/Sec: 1.10 + 57%|████████████████████████████████████████████████████████████████████████▍ | 25199/44547 [8:28:20<4:53:22, 1.10it/s][2025-03-24 16:57:03] (step=0025200) Train Loss: 3.8427, Train Steps/Sec: 1.05 + 57%|████████████████████████████████████████████████████████████████████████▍ | 25224/44547 [8:28:43<4:52:20, 1.10it/s][2025-03-24 16:57:26] (step=0025225) Train Loss: 3.8549, Train Steps/Sec: 1.10 + 57%|████████████████████████████████████████████████████████████████████████▌ | 25249/44547 [8:29:06<4:52:45, 1.10it/s][2025-03-24 16:57:48] (step=0025250) Train Loss: 3.7157, Train Steps/Sec: 1.10 + 57%|████████████████████████████████████████████████████████████████████████▌ | 25274/44547 [8:29:28<4:51:53, 1.10it/s][2025-03-24 16:58:11] (step=0025275) Train Loss: 3.8447, Train Steps/Sec: 1.10 + 57%|████████████████████████████████████████████████████████████████████████▋ | 25299/44547 [8:29:51<4:51:25, 1.10it/s][2025-03-24 16:58:34] (step=0025300) Train Loss: 3.7399, Train Steps/Sec: 1.10 + 57%|████████████████████████████████████████████████████████████████████████▊ | 25324/44547 [8:30:14<4:51:11, 1.10it/s][2025-03-24 16:58:57] (step=0025325) Train Loss: 3.7482, Train Steps/Sec: 1.10 + 57%|████████████████████████████████████████████████████████████████████████▊ | 25349/44547 [8:30:37<4:50:12, 1.10it/s][2025-03-24 16:59:19] (step=0025350) Train Loss: 3.8129, Train Steps/Sec: 1.10 + 57%|████████████████████████████████████████████████████████████████████████▉ | 25374/44547 [8:30:59<4:50:17, 1.10it/s][2025-03-24 16:59:42] (step=0025375) Train Loss: 3.7960, Train Steps/Sec: 1.10 + 57%|████████████████████████████████████████████████████████████████████████▉ | 25399/44547 [8:31:22<4:49:10, 1.10it/s][2025-03-24 17:00:05] (step=0025400) Train Loss: 3.6053, Train Steps/Sec: 1.10 + 57%|█████████████████████████████████████████████████████████████████████████ | 25424/44547 [8:31:45<4:48:37, 1.10it/s][2025-03-24 17:00:28] (step=0025425) Train Loss: 3.8660, Train Steps/Sec: 1.10 + 57%|█████████████████████████████████████████████████████████████████████████ | 25449/44547 [8:32:08<4:47:25, 1.11it/s][2025-03-24 17:00:50] (step=0025450) Train Loss: 3.8269, Train Steps/Sec: 1.10 + 57%|█████████████████████████████████████████████████████████████████████████▏ | 25474/44547 [8:32:30<4:49:09, 1.10it/s][2025-03-24 17:01:13] (step=0025475) Train Loss: 3.7993, Train Steps/Sec: 1.10 + 57%|█████████████████████████████████████████████████████████████████████████▎ | 25499/44547 [8:32:54<5:59:31, 1.13s/it][2025-03-24 17:01:37] (step=0025500) Train Loss: 3.7457, Train Steps/Sec: 1.05 + 57%|█████████████████████████████████████████████████████████████████████████▎ | 25524/44547 [8:33:17<4:48:19, 1.10it/s][2025-03-24 17:02:00] (step=0025525) Train Loss: 3.7921, Train Steps/Sec: 1.10 + 57%|█████████████████████████████████████████████████████████████████████████▍ | 25549/44547 [8:33:40<4:49:26, 1.09it/s][2025-03-24 17:02:23] (step=0025550) Train Loss: 3.6621, Train Steps/Sec: 1.10 + 57%|█████████████████████████████████████████████████████████████████████████▍ | 25574/44547 [8:34:04<4:53:49, 1.08it/s][2025-03-24 17:02:47] (step=0025575) Train Loss: 3.7383, Train Steps/Sec: 1.02 + 57%|█████████████████████████████████████████████████████████████████████████▌ | 25599/44547 [8:34:29<4:50:07, 1.09it/s][2025-03-24 17:03:12] (step=0025600) Train Loss: 3.7648, Train Steps/Sec: 0.99 + 58%|█████████████████████████████████████████████████████████████████████████▋ | 25624/44547 [8:34:52<4:48:36, 1.09it/s][2025-03-24 17:03:35] (step=0025625) Train Loss: 3.7981, Train Steps/Sec: 1.10 + 58%|█████████████████████████████████████████████████████████████████████████▋ | 25649/44547 [8:35:16<4:51:37, 1.08it/s][2025-03-24 17:03:59] (step=0025650) Train Loss: 3.7416, Train Steps/Sec: 1.05 + 58%|█████████████████████████████████████████████████████████████████████████▊ | 25674/44547 [8:35:39<4:49:12, 1.09it/s][2025-03-24 17:04:22] (step=0025675) Train Loss: 3.7311, Train Steps/Sec: 1.07 + 58%|█████████████████████████████████████████████████████████████████████████▊ | 25699/44547 [8:36:02<4:46:34, 1.10it/s][2025-03-24 17:04:45] (step=0025700) Train Loss: 3.7593, Train Steps/Sec: 1.10 + 58%|█████████████████████████████████████████████████████████████████████████▉ | 25714/44547 [8:36:16<4:45:52, 1.10it/s] diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/files/requirements.txt b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..d08a3e5e0c02054eefe2463e3487546131ecce79 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/files/requirements.txt @@ -0,0 +1,131 @@ +click==8.1.8 +networkx==3.3 +matplotlib-inline==0.1.7 +tqdm==4.67.1 +tzdata==2025.1 +pickleshare==0.7.5 +Jinja2==3.1.4 +opencv-python==4.11.0.86 +nvidia-cusparse-cu12==12.1.0.106 +fonttools==4.56.0 +ptyprocess==0.7.0 +nvidia-ml-py==12.570.86 +docker-pycreds==0.4.0 +datasets==3.4.0 +debugpy==1.8.13 +pillow==11.0.0 +charset-normalizer==3.4.1 +python-dateutil==2.9.0.post0 +wcwidth==0.2.13 +xxhash==3.5.0 +prompt_toolkit==3.0.50 +triton==3.1.0 +ipython==9.0.2 +aiosignal==1.3.2 +setuptools==75.8.0 +pyparsing==3.2.1 +tornado==6.4.2 +frozenlist==1.5.0 +requests==2.32.3 +parso==0.8.4 +MarkupSafe==2.1.5 +sentry-sdk==2.23.1 +comm==0.2.2 +contourpy==1.3.1 +certifi==2025.1.31 +propcache==0.3.0 +packaging==24.2 +idna==3.10 +nvitop==1.4.2 +pydantic_core==2.27.2 +pyarrow==19.0.1 +pexpect==4.9.0 +joblib==1.4.2 +fsspec==2024.12.0 +hf_transfer==0.1.9 +aiohttp==3.11.14 +wheel==0.45.1 +jupyter_client==8.6.3 +pytz==2025.1 +filelock==3.18.0 +multiprocess==0.70.16 +aiohappyeyeballs==2.6.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cublas-cu12==12.1.3.1 +importlib_metadata==8.6.1 +exceptiongroup==1.2.2 +urllib3==2.3.0 +tokenizers==0.21.1 +Pygments==2.19.1 +platformdirs==4.3.6 +mpmath==1.3.0 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-nvjitlink-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +regex==2024.11.6 +attrs==25.3.0 +asttokens==3.0.0 +pure_eval==0.2.3 +setproctitle==1.3.5 +kiwisolver==1.4.8 +nvidia-nccl-cu12==2.21.5 +torchaudio==2.5.1+cu121 +nvidia-cuda-nvrtc-cu12==12.1.105 +cycler==0.12.1 +PyYAML==6.0.2 +nest_asyncio==1.6.0 +jupyter_core==5.7.2 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +nvidia-cufft-cu12==11.0.2.54 +safetensors==0.5.3 +pip==25.0 +pyzmq==26.3.0 +smmap==5.0.2 +typing_extensions==4.12.2 +six==1.17.0 +torch==2.5.1+cu121 +zipp==3.21.0 +wandb==0.19.8 +gitdb==4.0.12 +decorator==5.2.1 +sympy==1.13.1 +traitlets==5.14.3 +numpy==2.2.4 +executing==2.1.0 +executing==2.2.0 +transformers==4.49.0 +protobuf==5.29.3 +annotated-types==0.7.0 +pandas==2.2.3 +jedi==0.19.2 +stack_data==0.6.3 +ipykernel==6.29.5 +psutil==7.0.0 +pydantic==2.10.6 +dill==0.3.8 +nltk==3.9.1 +multidict==6.1.0 +torchvision==0.20.1+cu121 +GitPython==3.1.44 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-nvtx-cu12==12.1.105 +nvidia-curand-cu12==10.3.2.106 +matplotlib==3.10.1 +huggingface-hub==0.29.3 +jaraco.text==3.12.1 +zipp==3.19.2 +jaraco.context==5.3.0 +autocommand==2.2.2 +inflect==7.3.1 +packaging==24.2 +jaraco.collections==5.1.0 +typeguard==4.3.0 +backports.tarfile==1.2.0 +tomli==2.0.1 +importlib_metadata==8.0.0 +typing_extensions==4.12.2 +jaraco.functools==4.0.1 +platformdirs==4.2.2 +wheel==0.43.0 +more-itertools==10.3.0 diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/files/wandb-metadata.json b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5e144d87ea3eb1ffac45afab7c8270f1898b01b2 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/files/wandb-metadata.json @@ -0,0 +1,149 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-03-24T08:28:41.054465Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_dir/subject_ti2i_t2i_stage1_w_flux_segment_mid_700k.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all", + "--cloud-save-path", + "/tmp/haozhezhao/MLLMG/checkpoints", + "--lr", + "5e-4", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_dir/new_1117_validation_set.jsonl_mid_1k.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/blip2-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "512", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "56", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "1", + "--train_text_encoder", + "--ckpt-every", + "4000", + "--epochs", + "4", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference.jsonl", + "--multimodal_encoder", + "llava", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--save_total_limit", + "1", + "--load_fixed_llamagen", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoints/llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_newfix_train_only_projection_i2i/014-GPT-XL/checkpoints/0022000.pt", + "--train_all" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "git": { + "remote": "git@github.com:HaozheZhao/MLLMG.git", + "commit": "f5d9894eac7b46fa35645a2cea9b04ee3b35f35b" + }, + "email": "mimazhe55360@gmail.com", + "root": "checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all", + "host": "447cc403a8794092814259713c51c1df00001D", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "73409888256" + } + }, + "memory": { + "total": "1902387908608" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug-core.log b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..2c3f02ab0f51138a01c3174f86cf7c368986598e --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug-core.log @@ -0,0 +1,6 @@ +{"time":"2025-03-24T08:28:40.554909483Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpf5xi1b71/port-1395084.txt","pid":1395084,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-24T08:28:40.556388508Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1395084} +{"time":"2025-03-24T08:28:40.556366138Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38817,"Zone":""}} +{"time":"2025-03-24T08:28:40.742460733Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:38934"} +{"time":"2025-03-24T08:28:41.055981609Z","level":"INFO","msg":"handleInformInit: received","streamId":"mlbv5b7s","id":"127.0.0.1:38934"} +{"time":"2025-03-24T08:28:41.189643817Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"mlbv5b7s","id":"127.0.0.1:38934"} diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug-internal.log b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f7dd89380c55af28d6ed794b69ba9a02c078c804 --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2025-03-24T08:28:41.056200986Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug-core.log"} +{"time":"2025-03-24T08:28:41.189607189Z","level":"INFO","msg":"created new stream","id":"mlbv5b7s"} +{"time":"2025-03-24T08:28:41.189638487Z","level":"INFO","msg":"stream: started","id":"mlbv5b7s"} +{"time":"2025-03-24T08:28:41.189791641Z","level":"INFO","msg":"handler: started","stream_id":"mlbv5b7s"} +{"time":"2025-03-24T08:28:41.189819323Z","level":"INFO","msg":"writer: Do: started","stream_id":"mlbv5b7s"} +{"time":"2025-03-24T08:28:41.18982348Z","level":"INFO","msg":"sender: started","stream_id":"mlbv5b7s"} +{"time":"2025-03-24T08:28:41.484118777Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-24T17:02:13.70574757Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug.log b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b78bf9b6f3eed15fe248ca7684be0cfaa2bcb32b --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug.log @@ -0,0 +1,22 @@ +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_setup.py:_flush():67] Configure stats pid to 1395084 +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug.log +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/logs/debug-internal.log +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_init.py:init():761] calling init triggers +2025-03-24 08:28:41,048 INFO MainThread:1395084 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_dir/subject_ti2i_t2i_stage1_w_flux_segment_mid_700k.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoints', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoints/llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_newfix_train_only_projection_i2i/014-GPT-XL/checkpoints/0022000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 4, 'lr': 0.0005, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 4000, 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_dir/new_1117_validation_set.jsonl_mid_1k.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 512, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': False, 'no_replace': False, 'resume': False, 'dreambench_eval': False, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'i2i': False, 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-03-24 08:28:41,049 INFO MainThread:1395084 [wandb_init.py:init():784] starting backend +2025-03-24 08:28:41,049 INFO MainThread:1395084 [wandb_init.py:init():788] sending inform_init request +2025-03-24 08:28:41,054 INFO MainThread:1395084 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-03-24 08:28:41,054 INFO MainThread:1395084 [wandb_init.py:init():798] backend started and connected +2025-03-24 08:28:41,056 INFO MainThread:1395084 [wandb_init.py:init():891] updated telemetry +2025-03-24 08:28:41,087 INFO MainThread:1395084 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-03-24 08:28:41,481 INFO MainThread:1395084 [wandb_init.py:init():990] starting run threads in backend +2025-03-24 08:28:41,560 INFO MainThread:1395084 [wandb_run.py:_console_start():2375] atexit reg +2025-03-24 08:28:41,561 INFO MainThread:1395084 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-03-24 08:28:41,562 INFO MainThread:1395084 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-03-24 08:28:41,562 INFO MainThread:1395084 [wandb_run.py:_redirect():2315] Redirects installed. +2025-03-24 08:28:41,563 INFO MainThread:1395084 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/run-mlbv5b7s.wandb b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/run-mlbv5b7s.wandb new file mode 100644 index 0000000000000000000000000000000000000000..83b170119f521de18390c6088c66cf38c94e440f --- /dev/null +++ b/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/wandb/run-20250324_082841-mlbv5b7s/run-mlbv5b7s.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0490d24e1c53ad65cbac5d1eff03a46135486db260572c3fbb47eb2ccda44f3 +size 15958016