Training in progress, step 10750, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d437bc70c4c04c351d9b268de8b986e9c10960030d11d9b5cc0a07a032d4e75
+oid sha256:93a21c4fe29a9103dca860aa3bb15c0e288b74846b4a59c29b3fd8a0bf27740b
 size 29034840

last-checkpoint/global_step10750/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43e2601f2d3c98983b4ee43fa2686c00b346705e83fc3ea0a86bc3417b5d420c
+size 43429616

last-checkpoint/global_step10750/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31b687950ea88705f4a6f202a38ff6ac1e7f9cd3e488cd1dc631c5bc9f940da5
+size 43429616

last-checkpoint/global_step10750/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7fa7bc5f002e6adaf92b93e1b55e50c26081ee1ce65671b97ac17ca5fca3226
+size 43429616

last-checkpoint/global_step10750/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:271a22975994b10de58033da1bc3c5226bf30c17750f50f91d72fe94c629e664
+size 43429616

last-checkpoint/global_step10750/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9861c41f1725f2f024985cc75b883aee43d7c1e94a0df5bd1f8fbbc3450946c2
+size 637299

last-checkpoint/global_step10750/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8d3b758679140b5987ca75db93e8f9770288ea2b70a76fa1f38887c1e209697
+size 637171

last-checkpoint/global_step10750/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:098ea93049be2fbec3368ad9527cda183fab040b92eb46206600628c4d8e6a97
+size 637171

last-checkpoint/global_step10750/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9efc89e450a48f389ff9fa07b9ba567e69ecd990abac861b572e76644f9411e5
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step10650
+global_step10750

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:882705638474934670faaaf11f480a9ca965116088d43b545d3c16f14930bd88
+oid sha256:4c08e0df7f2c84b8164ab107eb8d3cd4a985cd82fe5a60ae9e364a68ec3639a3
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b49e541bf1b6ee313511bcb33f9c57c56d53d3ca5fc5060ada8ae009ad408c0
+oid sha256:55edbc753eb49d62368d11f179521fb9d5ae7a41bf3df439be5b58898588d862
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c575f73a7906ffee30747f33b3ff9d606e71678d1c70f907532bff0706c91b26
+oid sha256:389ea1a6d6727e263f6a278450853d396adad7f849ee20ba1051d7524e720482
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad85b3229606911aae4c7db32cce537b0b9493ac5a6f9cb5cde256955cbe0543
+oid sha256:5d23b87e78b4bd2aa5e9af52b52401821aafa218a5f4b2f71fc3a30e0dd77c67
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:284555ebd24f2f7c70a3cb7790c9d4c47d92d96814317f7156c53c32553482a5
+oid sha256:81f6af293e8bd78f46769a700265966fbd8048df89a7524d9c0b66db799d7a8e
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.04092838987708092,
   "best_model_checkpoint": "saves/CADICA_qwenvl_detect_classify_augmented/lora/sft/checkpoint-9050",
-  "epoch": 1.3047992404520534,
+  "epoch": 1.31705001378212,
   "eval_steps": 50,
-  "global_step": 10650,
+  "global_step": 10750,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -18964,11 +18964,189 @@
       "eval_steps_per_second": 0.768,
       "num_input_tokens_seen": 71167464,
       "step": 10650
+    },
+    {
+      "epoch": 1.3054117791185569,
+      "grad_norm": 1.206825055856801,
+      "learning_rate": 2.950689735821428e-05,
+      "loss": 0.2511,
+      "num_input_tokens_seen": 71200776,
+      "step": 10655
+    },
+    {
+      "epoch": 1.3060243177850601,
+      "grad_norm": 0.8738688843235899,
+      "learning_rate": 2.9460709454307668e-05,
+      "loss": 0.2303,
+      "num_input_tokens_seen": 71234168,
+      "step": 10660
+    },
+    {
+      "epoch": 1.3066368564515636,
+      "grad_norm": 1.2670653369653868,
+      "learning_rate": 2.9414542625498032e-05,
+      "loss": 0.2423,
+      "num_input_tokens_seen": 71268008,
+      "step": 10665
+    },
+    {
+      "epoch": 1.3072493951180668,
+      "grad_norm": 1.0765877081535313,
+      "learning_rate": 2.9368396919156543e-05,
+      "loss": 0.2461,
+      "num_input_tokens_seen": 71301256,
+      "step": 10670
+    },
+    {
+      "epoch": 1.30786193378457,
+      "grad_norm": 1.0930641133259775,
+      "learning_rate": 2.932227238263271e-05,
+      "loss": 0.2585,
+      "num_input_tokens_seen": 71334304,
+      "step": 10675
+    },
+    {
+      "epoch": 1.3084744724510735,
+      "grad_norm": 1.5727229135478251,
+      "learning_rate": 2.927616906325432e-05,
+      "loss": 0.2558,
+      "num_input_tokens_seen": 71367784,
+      "step": 10680
+    },
+    {
+      "epoch": 1.3090870111175767,
+      "grad_norm": 1.1094592860069892,
+      "learning_rate": 2.9230087008327368e-05,
+      "loss": 0.2396,
+      "num_input_tokens_seen": 71400960,
+      "step": 10685
+    },
+    {
+      "epoch": 1.3096995497840802,
+      "grad_norm": 1.248724168955527,
+      "learning_rate": 2.9184026265136056e-05,
+      "loss": 0.2212,
+      "num_input_tokens_seen": 71433712,
+      "step": 10690
+    },
+    {
+      "epoch": 1.3103120884505834,
+      "grad_norm": 1.7911517740511143,
+      "learning_rate": 2.9137986880942715e-05,
+      "loss": 0.2074,
+      "num_input_tokens_seen": 71467256,
+      "step": 10695
+    },
+    {
+      "epoch": 1.3109246271170867,
+      "grad_norm": 1.2413107560000207,
+      "learning_rate": 2.909196890298773e-05,
+      "loss": 0.206,
+      "num_input_tokens_seen": 71501128,
+      "step": 10700
+    },
+    {
+      "epoch": 1.3109246271170867,
+      "eval_loss": 0.19016502797603607,
+      "eval_runtime": 19.7377,
+      "eval_samples_per_second": 3.04,
+      "eval_steps_per_second": 0.76,
+      "num_input_tokens_seen": 71501128,
+      "step": 10700
+    },
+    {
+      "epoch": 1.3115371657835901,
+      "grad_norm": 1.440592474356754,
+      "learning_rate": 2.9045972378489604e-05,
+      "loss": 0.2671,
+      "num_input_tokens_seen": 71534760,
+      "step": 10705
+    },
+    {
+      "epoch": 1.3121497044500934,
+      "grad_norm": 1.6043559122436624,
+      "learning_rate": 2.8999997354644698e-05,
+      "loss": 0.2431,
+      "num_input_tokens_seen": 71568000,
+      "step": 10710
+    },
+    {
+      "epoch": 1.3127622431165968,
+      "grad_norm": 1.6562341112618186,
+      "learning_rate": 2.8954043878627414e-05,
+      "loss": 0.2767,
+      "num_input_tokens_seen": 71601664,
+      "step": 10715
+    },
+    {
+      "epoch": 1.3133747817831,
+      "grad_norm": 1.3116654592131844,
+      "learning_rate": 2.890811199759003e-05,
+      "loss": 0.2077,
+      "num_input_tokens_seen": 71635832,
+      "step": 10720
+    },
+    {
+      "epoch": 1.3139873204496033,
+      "grad_norm": 13.474160151030459,
+      "learning_rate": 2.886220175866261e-05,
+      "loss": 0.2503,
+      "num_input_tokens_seen": 71669720,
+      "step": 10725
+    },
+    {
+      "epoch": 1.3145998591161068,
+      "grad_norm": 1.3031307301091186,
+      "learning_rate": 2.881631320895306e-05,
+      "loss": 0.2579,
+      "num_input_tokens_seen": 71703304,
+      "step": 10730
+    },
+    {
+      "epoch": 1.31521239778261,
+      "grad_norm": 1.193828203752651,
+      "learning_rate": 2.8770446395547036e-05,
+      "loss": 0.2089,
+      "num_input_tokens_seen": 71737192,
+      "step": 10735
+    },
+    {
+      "epoch": 1.3158249364491135,
+      "grad_norm": 1.0929056779676314,
+      "learning_rate": 2.87246013655079e-05,
+      "loss": 0.2317,
+      "num_input_tokens_seen": 71770632,
+      "step": 10740
+    },
+    {
+      "epoch": 1.3164374751156167,
+      "grad_norm": 1.0813012821730756,
+      "learning_rate": 2.8678778165876556e-05,
+      "loss": 0.2344,
+      "num_input_tokens_seen": 71804112,
+      "step": 10745
+    },
+    {
+      "epoch": 1.31705001378212,
+      "grad_norm": 1.228614756425054,
+      "learning_rate": 2.863297684367164e-05,
+      "loss": 0.184,
+      "num_input_tokens_seen": 71837520,
+      "step": 10750
+    },
+    {
+      "epoch": 1.31705001378212,
+      "eval_loss": 0.2411387860774994,
+      "eval_runtime": 19.9995,
+      "eval_samples_per_second": 3.0,
+      "eval_steps_per_second": 0.75,
+      "num_input_tokens_seen": 71837520,
+      "step": 10750
     }
   ],
   "logging_steps": 5,
   "max_steps": 16324,
-  "num_input_tokens_seen": 71167464,
+  "num_input_tokens_seen": 71837520,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -18983,7 +19161,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4464001544224768.0,
+  "total_flos": 4505992934326272.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null