Training in progress, step 1000, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:000a3ba372c75830fe0eb4f957d1f2ca218684806449e54609e0939d1f6995de
+oid sha256:d0f9be3d791b991c20113ef9cf18bf0eacc627445e0c958d7fcaece9aee60abe
 size 29034840

last-checkpoint/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b08bbdb7efeb127b4a831913f37ebc0497df8e82a0f53eacd9098c3ec16153c
+size 43429616

last-checkpoint/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e30009a7de062461354f3f268b368f5284b41d1185efca5f61d78882130c6c3
+size 43429616

last-checkpoint/global_step1000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbdd9edb4b9ca4a3ab1270601c55046401095638286819172192ab1088568fdd
+size 43429616

last-checkpoint/global_step1000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19a1de16478856a65e420eb14973cda96aff13a3ea4dd378d9b3e06ac38b2326
+size 43429616

last-checkpoint/global_step1000/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c9bd391981c97acfa81f9157218f91938449621f6977a56553d555f3f51a88d
+size 637299

last-checkpoint/global_step1000/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85a69064974a1a815ff8d2ed070c37b1f613adba7988ac39224738277b6d7ddd
+size 637171

last-checkpoint/global_step1000/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a742ccd3b52b2bdc6d240702273e583950d659c6f71515b868c1e36b6def99e5
+size 637171

last-checkpoint/global_step1000/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67279beb08daa878a7fc70b5ac88cabc029f948a379a271bbb43983d710bea32
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step950
+global_step1000

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f2439da621f14c22b4f733e91bfc9de6b506d28d7b8d6f3eaca2e0b4f24c078
+oid sha256:7d73dfcc09cf3d6f08149535e03920234febc15f7e9a166987f3bc01ee871abf
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9e3fb386557f376b8946af5b8c91f9418f374dddb2ad9da4868b1ef16778c32
+oid sha256:4396a64b6da4868d060d1e3c7c9ccb12c39d63bd0f7b146d2512400aff4c769c
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc7774d06045635bece9e960378fdc6913bf7bbbc903444cc570d1ca6ac25645
+oid sha256:95877efc8fb5eb302819ee7effca4222569cdcfdebb9fa5d9846e68ed9e833fe
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d98c54a80a914fecf43d06ea81432499f46e70664f1d04651bf339163e30fa9e
+oid sha256:e9fa4f23377f00fdde731da68a8690098617a1fdd912e03cdaa8bde87c493179
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9f224baf5bd2044314606c1d88f84cce32f1b37c43c15835b14e72f6a72a4fc
+oid sha256:bce7739c5bb5cf50e8f1c942e662e33e6aa589036d55e6fddd63bdf3171c1cae
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6621683239936829,
   "best_model_checkpoint": "saves/CADICA_qwenvl_detect_classify_augmented/lora/sft/checkpoint-900",
-  "epoch": 0.11638234663563138,
+  "epoch": 0.1225077333006646,
   "eval_steps": 50,
-  "global_step": 950,
+  "global_step": 1000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1698,11 +1698,100 @@
       "eval_steps_per_second": 0.781,
       "num_input_tokens_seen": 6544656,
       "step": 950
+    },
+    {
+      "epoch": 0.1169948853021347,
+      "grad_norm": 7.122796737825454,
+      "learning_rate": 8.612053338817581e-05,
+      "loss": 0.5924,
+      "num_input_tokens_seen": 6578488,
+      "step": 955
+    },
+    {
+      "epoch": 0.11760742396863802,
+      "grad_norm": 4.508377480122389,
+      "learning_rate": 8.595197232161824e-05,
+      "loss": 0.5946,
+      "num_input_tokens_seen": 6611616,
+      "step": 960
+    },
+    {
+      "epoch": 0.11821996263514134,
+      "grad_norm": 5.733858610554117,
+      "learning_rate": 8.578256098561275e-05,
+      "loss": 0.5692,
+      "num_input_tokens_seen": 6645456,
+      "step": 965
+    },
+    {
+      "epoch": 0.11883250130164466,
+      "grad_norm": 1.904868426487018,
+      "learning_rate": 8.561230338676239e-05,
+      "loss": 0.5347,
+      "num_input_tokens_seen": 6678648,
+      "step": 970
+    },
+    {
+      "epoch": 0.119445039968148,
+      "grad_norm": 11.713013089175845,
+      "learning_rate": 8.544120355168451e-05,
+      "loss": 0.5792,
+      "num_input_tokens_seen": 6712928,
+      "step": 975
+    },
+    {
+      "epoch": 0.12005757863465132,
+      "grad_norm": 6.617168404497372,
+      "learning_rate": 8.526926552691544e-05,
+      "loss": 0.5768,
+      "num_input_tokens_seen": 6746616,
+      "step": 980
+    },
+    {
+      "epoch": 0.12067011730115464,
+      "grad_norm": 3.1162749994283203,
+      "learning_rate": 8.509649337881483e-05,
+      "loss": 0.53,
+      "num_input_tokens_seen": 6779552,
+      "step": 985
+    },
+    {
+      "epoch": 0.12128265596765796,
+      "grad_norm": 4.612530110988974,
+      "learning_rate": 8.492289119346943e-05,
+      "loss": 0.5095,
+      "num_input_tokens_seen": 6812968,
+      "step": 990
+    },
+    {
+      "epoch": 0.12189519463416128,
+      "grad_norm": 8.47574836104151,
+      "learning_rate": 8.474846307659658e-05,
+      "loss": 0.5278,
+      "num_input_tokens_seen": 6846816,
+      "step": 995
+    },
+    {
+      "epoch": 0.1225077333006646,
+      "grad_norm": 21.010943165658347,
+      "learning_rate": 8.457321315344694e-05,
+      "loss": 0.55,
+      "num_input_tokens_seen": 6880016,
+      "step": 1000
+    },
+    {
+      "epoch": 0.1225077333006646,
+      "eval_loss": 0.7234537601470947,
+      "eval_runtime": 19.3824,
+      "eval_samples_per_second": 3.096,
+      "eval_steps_per_second": 0.774,
+      "num_input_tokens_seen": 6880016,
+      "step": 1000
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 6544656,
+  "num_input_tokens_seen": 6880016,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -1717,7 +1806,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 411865367183360.0,
+  "total_flos": 432927769755648.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null