End of training

README.md

@@ -5,10 +5,10 @@ model_name: Qwen2.5-0.5B-Instruct-Gensyn-Swarm-quick_timid_frog
 tags:
 - generated_from_trainer
 - rl-swarm
-- trl
-- I am quick timid frog
 - gensyn
 - grpo
+- I am quick timid frog
+- trl
 licence: license
 ---
 

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.05050143450498581,
-    "train_runtime": 733.4022,
+    "train_loss": 0.0874896764755249,
+    "train_runtime": 937.823,
     "train_samples": 5,
-    "train_samples_per_second": 0.218,
-    "train_steps_per_second": 0.027
+    "train_samples_per_second": 0.171,
+    "train_steps_per_second": 0.021
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a00a688a4a482c3644a8ebeecdb6df4ad1b67cdeca0702cf7ef00d4ca1271b5
+oid sha256:713264c96749c24604d08d142b497451f9725614fb94d291a0e5e9dee6936d71
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.05050143450498581,
-    "train_runtime": 733.4022,
+    "train_loss": 0.0874896764755249,
+    "train_runtime": 937.823,
     "train_samples": 5,
-    "train_samples_per_second": 0.218,
-    "train_steps_per_second": 0.027
+    "train_samples_per_second": 0.171,
+    "train_steps_per_second": 0.021
 }

trainer_state.json

@@ -15,22 +15,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 736.5,
-      "completions/max_terminated_length": 517.5,
-      "completions/mean_length": 234.125,
-      "completions/mean_terminated_length": 181.95536041259766,
-      "completions/min_length": 13.5,
-      "completions/min_terminated_length": 13.5,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 938.0,
+      "completions/max_terminated_length": 865.5,
+      "completions/mean_length": 656.125,
+      "completions/mean_terminated_length": 580.375,
+      "completions/min_length": 287.5,
+      "completions/min_terminated_length": 287.5,
       "epoch": 1.8,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 9.468099594116211,
-      "kl": 0.0009500358792138286,
+      "grad_norm": 2.378174066543579,
+      "kl": 0.00024694142030057264,
       "learning_rate": 5e-07,
-      "loss": 0.1074,
-      "num_tokens": 7842.0,
-      "reward": 0.14829658716917038,
-      "reward_std": 0.09615837037563324,
+      "loss": -0.0692,
+      "num_tokens": 14594.0,
+      "reward": -0.09812777116894722,
+      "reward_std": 0.16612516529858112,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -39,14 +39,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.1441090926527977,
-      "rewards/question_recreation_reward_func/std": 0.0952618196606636,
+      "rewards/question_recreation_reward_func/mean": 0.01712222769856453,
+      "rewards/question_recreation_reward_func/std": 0.01072776922956109,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.004187500104308128,
-      "rewards/xmlcount_reward_func/std": 0.011844038963317871,
+      "rewards/xmlcount_reward_func/mean": -0.11524999886751175,
+      "rewards/xmlcount_reward_func/std": 0.32597625255584717,
       "step": 2
     },
     {
@@ -57,20 +57,20 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.125,
       "completions/max_length": 1024.0,
-      "completions/max_terminated_length": 787.5,
-      "completions/mean_length": 421.5,
-      "completions/mean_terminated_length": 335.4285888671875,
-      "completions/min_length": 18.0,
-      "completions/min_terminated_length": 18.0,
+      "completions/max_terminated_length": 868.0,
+      "completions/mean_length": 470.0625,
+      "completions/mean_terminated_length": 390.9285888671875,
+      "completions/min_length": 60.0,
+      "completions/min_terminated_length": 60.0,
       "epoch": 3.8,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 4.018620491027832,
-      "kl": 0.0006310820899670944,
+      "grad_norm": 5.8317060470581055,
+      "kl": 0.00036002036267746007,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.1916,
-      "num_tokens": 18682.0,
-      "reward": 0.022797180339694023,
-      "reward_std": 0.007740819826722145,
+      "loss": 0.136,
+      "num_tokens": 26211.0,
+      "reward": 0.07699444144964218,
+      "reward_std": 0.030455347150564194,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -79,8 +79,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.022797180339694023,
-      "rewards/question_recreation_reward_func/std": 0.015481723938137293,
+      "rewards/question_recreation_reward_func/mean": 0.07699444144964218,
+      "rewards/question_recreation_reward_func/std": 0.04443136602640152,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -95,22 +95,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.1875,
+      "completions/clipped_ratio": 0.125,
       "completions/max_length": 1024.0,
-      "completions/max_terminated_length": 836.5,
-      "completions/mean_length": 406.625,
-      "completions/mean_terminated_length": 263.8690643310547,
-      "completions/min_length": 8.5,
-      "completions/min_terminated_length": 8.5,
+      "completions/max_terminated_length": 1003.5,
+      "completions/mean_length": 548.625,
+      "completions/mean_terminated_length": 480.7143096923828,
+      "completions/min_length": 134.0,
+      "completions/min_terminated_length": 134.0,
       "epoch": 5.8,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 9.984585762023926,
-      "kl": 0.0017272870391025208,
+      "grad_norm": 3.393465518951416,
+      "kl": 0.00036590506533684675,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.1226,
-      "num_tokens": 29284.0,
-      "reward": 0.05447566136717796,
-      "reward_std": 0.040898644365370274,
+      "loss": 0.1056,
+      "num_tokens": 39085.0,
+      "reward": 0.052806172519922256,
+      "reward_std": 0.017434499226510525,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -119,8 +119,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.05447566136717796,
-      "rewards/question_recreation_reward_func/std": 0.03653379296883941,
+      "rewards/question_recreation_reward_func/mean": 0.052806172519922256,
+      "rewards/question_recreation_reward_func/std": 0.018543646670877934,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -135,22 +135,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.25,
-      "completions/max_length": 1024.0,
-      "completions/max_terminated_length": 389.5,
-      "completions/mean_length": 363.3125,
-      "completions/mean_terminated_length": 143.08333587646484,
-      "completions/min_length": 17.0,
-      "completions/min_terminated_length": 17.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 900.0,
+      "completions/max_terminated_length": 900.0,
+      "completions/mean_length": 458.8125,
+      "completions/mean_terminated_length": 458.8125,
+      "completions/min_length": 114.0,
+      "completions/min_terminated_length": 114.0,
       "epoch": 7.8,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 9.14108657836914,
-      "kl": 0.006233984004211379,
+      "grad_norm": 4.418057918548584,
+      "kl": 0.0005609585878119105,
       "learning_rate": 3.867370395306068e-07,
-      "loss": -0.1465,
-      "num_tokens": 39193.0,
-      "reward": 0.09087881818413734,
-      "reward_std": 0.06694184988737106,
+      "loss": 0.0939,
+      "num_tokens": 50522.0,
+      "reward": 0.024604666978120804,
+      "reward_std": 0.009030941408127546,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -159,14 +159,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.1174413226544857,
-      "rewards/question_recreation_reward_func/std": 0.07382831908762455,
+      "rewards/question_recreation_reward_func/mean": 0.024604666978120804,
+      "rewards/question_recreation_reward_func/std": 0.011531331343576312,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": -0.026562499813735485,
-      "rewards/xmlcount_reward_func/std": 0.07513009570538998,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 8
     },
     {
@@ -175,22 +175,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.25,
-      "completions/max_length": 1024.0,
-      "completions/max_terminated_length": 272.0,
-      "completions/mean_length": 344.125,
-      "completions/mean_terminated_length": 117.50000381469727,
-      "completions/min_length": 30.5,
-      "completions/min_terminated_length": 30.5,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 928.5,
+      "completions/max_terminated_length": 824.0,
+      "completions/mean_length": 451.25,
+      "completions/mean_terminated_length": 412.8482208251953,
+      "completions/min_length": 56.0,
+      "completions/min_terminated_length": 56.0,
       "epoch": 9.8,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 8.28457260131836,
-      "kl": 0.005484735171194188,
+      "grad_norm": 4.358504772186279,
+      "kl": 0.0006784129072912037,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": -0.0226,
-      "num_tokens": 48795.0,
-      "reward": 0.09757254645228386,
-      "reward_std": 0.05229111574590206,
+      "loss": 0.1711,
+      "num_tokens": 61838.0,
+      "reward": 0.08084426820278168,
+      "reward_std": 0.05193536356091499,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -199,29 +199,29 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.0833225455135107,
-      "rewards/question_recreation_reward_func/std": 0.056736329570412636,
+      "rewards/question_recreation_reward_func/mean": 0.08084426820278168,
+      "rewards/question_recreation_reward_func/std": 0.08748885244131088,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.01425000000745058,
-      "rewards/xmlcount_reward_func/std": 0.040305085480213165,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 10
     },
     {
       "epoch": 9.8,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 0.05050143450498581,
-      "train_runtime": 733.4022,
-      "train_samples_per_second": 0.218,
-      "train_steps_per_second": 0.027
+      "train_loss": 0.0874896764755249,
+      "train_runtime": 937.823,
+      "train_samples_per_second": 0.171,
+      "train_steps_per_second": 0.021
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
-  "num_input_tokens_seen": 48795,
+  "num_input_tokens_seen": 61838,
   "num_train_epochs": 10,
   "save_steps": 25,
   "stateful_callbacks": {