End of training

README.md

@@ -4,11 +4,11 @@ library_name: transformers
 model_name: Qwen2.5-0.5B-Instruct-Gensyn-Swarm-quick_timid_frog
 tags:
 - generated_from_trainer
-- grpo
 - gensyn
-- I am quick timid frog
 - trl
 - rl-swarm
+- I am quick timid frog
+- grpo
 licence: license
 ---
 

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.15644574165344238,
-    "train_runtime": 1523.8731,
-    "train_samples": 79,
-    "train_samples_per_second": 0.105,
+    "train_loss": 0.0773889608681202,
+    "train_runtime": 1482.6063,
+    "train_samples": 100,
+    "train_samples_per_second": 0.108,
     "train_steps_per_second": 0.013
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24593988fb48726d4a5db5e2e37fe74d6a678e8b4785c2c95c767b747a674505
+oid sha256:ec1757c7bcfb69a7ca1f90936efaae4cbde51cad9aa6283191230f67cfe9dd0a
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.15644574165344238,
-    "train_runtime": 1523.8731,
-    "train_samples": 79,
-    "train_samples_per_second": 0.105,
+    "train_loss": 0.0773889608681202,
+    "train_runtime": 1482.6063,
+    "train_samples": 100,
+    "train_samples_per_second": 0.108,
     "train_steps_per_second": 0.013
 }

trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9620253164556962,
+  "epoch": 0.8,
   "eval_steps": 500,
-  "global_step": 19,
+  "global_step": 20,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -16,21 +16,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.125,
-      "completions/max_length": 931.0,
-      "completions/max_terminated_length": 819.5,
-      "completions/mean_length": 412.1875,
-      "completions/mean_terminated_length": 329.5833435058594,
-      "completions/min_length": 87.0,
-      "completions/min_terminated_length": 87.0,
-      "epoch": 0.10126582278481013,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 605.0,
+      "completions/mean_length": 324.4375,
+      "completions/mean_terminated_length": 224.50000762939453,
+      "completions/min_length": 5.5,
+      "completions/min_terminated_length": 5.5,
+      "epoch": 0.08,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 5.04105806350708,
-      "kl": -5.448857809398078e-09,
+      "grad_norm": 9.534358024597168,
+      "kl": -5.265982289115456e-09,
       "learning_rate": 5e-07,
-      "loss": 0.1398,
-      "num_tokens": 10691.0,
-      "reward": 0.01983704511076212,
-      "reward_std": 0.008039223263040185,
+      "loss": 0.0881,
+      "num_tokens": 9287.0,
+      "reward": 0.02333822101354599,
+      "reward_std": 0.011800897307693958,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -39,8 +39,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.01983704511076212,
-      "rewards/question_recreation_reward_func/std": 0.012273336760699749,
+      "rewards/question_recreation_reward_func/mean": 0.02333822101354599,
+      "rewards/question_recreation_reward_func/std": 0.014200002886354923,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -56,21 +56,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.125,
-      "completions/max_length": 1024.0,
-      "completions/max_terminated_length": 569.0,
-      "completions/mean_length": 401.1875,
-      "completions/mean_terminated_length": 312.21429443359375,
-      "completions/min_length": 53.5,
-      "completions/min_terminated_length": 53.5,
-      "epoch": 0.20253164556962025,
+      "completions/max_length": 831.5,
+      "completions/max_terminated_length": 502.5,
+      "completions/mean_length": 283.4375,
+      "completions/mean_terminated_length": 169.41666793823242,
+      "completions/min_length": 14.0,
+      "completions/min_terminated_length": 14.0,
+      "epoch": 0.16,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 3.7304916381835938,
-      "kl": 0.0002279730260852375,
+      "grad_norm": 12.92307186126709,
+      "kl": 0.0008189797645172803,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.1637,
-      "num_tokens": 21206.0,
-      "reward": -0.1395124290138483,
-      "reward_std": 0.21926994435489178,
+      "loss": -0.0098,
+      "num_tokens": 17918.0,
+      "reward": 0.08256983105093241,
+      "reward_std": 0.01837824168615043,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -79,14 +79,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.010925075970590115,
-      "rewards/question_recreation_reward_func/std": 0.009428349556401372,
+      "rewards/question_recreation_reward_func/mean": 0.08256983105093241,
+      "rewards/question_recreation_reward_func/std": 0.023414009949192405,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": -0.15043750405311584,
-      "rewards/xmlcount_reward_func/std": 0.4255015254020691,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 4
     },
     {
@@ -95,22 +95,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 887.0,
-      "completions/max_terminated_length": 672.5,
-      "completions/mean_length": 362.0,
-      "completions/mean_terminated_length": 257.3125,
-      "completions/min_length": 12.5,
-      "completions/min_terminated_length": 12.5,
-      "epoch": 0.3037974683544304,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 722.5,
+      "completions/max_terminated_length": 722.5,
+      "completions/mean_length": 253.75,
+      "completions/mean_terminated_length": 253.75,
+      "completions/min_length": 24.5,
+      "completions/min_terminated_length": 24.5,
+      "epoch": 0.24,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 7.230990886688232,
-      "kl": 0.0008080850275291596,
+      "grad_norm": 6.703396320343018,
+      "kl": 0.0009663624296081252,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.2574,
-      "num_tokens": 31094.0,
-      "reward": 0.022412247024476528,
-      "reward_std": 0.011643366422504187,
+      "loss": 0.159,
+      "num_tokens": 26074.0,
+      "reward": 0.026149642653763294,
+      "reward_std": 0.01439021248370409,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -119,8 +119,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.022412247024476528,
-      "rewards/question_recreation_reward_func/std": 0.01874966360628605,
+      "rewards/question_recreation_reward_func/mean": 0.026149642653763294,
+      "rewards/question_recreation_reward_func/std": 0.01520916074514389,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -135,22 +135,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.1875,
-      "completions/max_length": 862.5,
-      "completions/max_terminated_length": 702.5,
-      "completions/mean_length": 461.6875,
-      "completions/mean_terminated_length": 357.3249969482422,
-      "completions/min_length": 67.0,
-      "completions/min_terminated_length": 67.0,
-      "epoch": 0.4050632911392405,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 678.5,
+      "completions/max_terminated_length": 678.5,
+      "completions/mean_length": 289.75,
+      "completions/mean_terminated_length": 289.75,
+      "completions/min_length": 70.5,
+      "completions/min_terminated_length": 70.5,
+      "epoch": 0.32,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 3.539825677871704,
-      "kl": 0.0011349972373864148,
+      "grad_norm": 7.256834030151367,
+      "kl": 0.0016369151817343663,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.2018,
-      "num_tokens": 42577.0,
-      "reward": 0.10562402009963989,
-      "reward_std": 0.061309706419706345,
+      "loss": -0.0972,
+      "num_tokens": 34806.0,
+      "reward": 0.060055448208004236,
+      "reward_std": 0.020912725245580077,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -159,8 +159,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.10562402009963989,
-      "rewards/question_recreation_reward_func/std": 0.05985743924975395,
+      "rewards/question_recreation_reward_func/mean": 0.060055448208004236,
+      "rewards/question_recreation_reward_func/std": 0.034508606884628534,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -175,32 +175,32 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 664.0,
-      "completions/max_terminated_length": 436.5,
-      "completions/mean_length": 222.4375,
-      "completions/mean_terminated_length": 173.0357208251953,
-      "completions/min_length": 4.5,
-      "completions/min_terminated_length": 4.5,
-      "epoch": 0.5063291139240507,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 555.5,
+      "completions/max_terminated_length": 555.5,
+      "completions/mean_length": 258.375,
+      "completions/mean_terminated_length": 258.375,
+      "completions/min_length": 3.5,
+      "completions/min_terminated_length": 3.5,
+      "epoch": 0.4,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 13.28281021118164,
-      "kl": 0.0055584801666554995,
+      "grad_norm": 5.563640594482422,
+      "kl": 0.0017646014493948314,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0764,
-      "num_tokens": 50232.0,
-      "reward": 0.05675883777439594,
-      "reward_std": 0.017147527541965246,
+      "loss": 0.1804,
+      "num_tokens": 43036.0,
+      "reward": 0.1413715137168765,
+      "reward_std": 0.18358006980270147,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.125,
+      "rewards/consensus_reward_func/std": 0.3535533845424652,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.05675883777439594,
-      "rewards/question_recreation_reward_func/std": 0.02832796238362789,
+      "rewards/question_recreation_reward_func/mean": 0.016371519304811954,
+      "rewards/question_recreation_reward_func/std": 0.011078037787228823,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -215,32 +215,32 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 681.5,
-      "completions/max_terminated_length": 319.0,
-      "completions/mean_length": 298.25,
-      "completions/mean_terminated_length": 195.8541717529297,
-      "completions/min_length": 73.5,
-      "completions/min_terminated_length": 73.5,
-      "epoch": 0.6075949367088608,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 670.0,
+      "completions/max_terminated_length": 533.5,
+      "completions/mean_length": 258.75,
+      "completions/mean_terminated_length": 214.0,
+      "completions/min_length": 4.5,
+      "completions/min_terminated_length": 4.5,
+      "epoch": 0.48,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 8.755450248718262,
-      "kl": 0.0035302894830238074,
+      "grad_norm": 6.797975063323975,
+      "kl": 0.003329606697661802,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.2667,
-      "num_tokens": 59100.0,
-      "reward": 0.015352241694927216,
-      "reward_std": 0.0057732411660254,
-      "rewards/concensus_correctness_reward_func/mean": 0.0,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "loss": 0.0231,
+      "num_tokens": 51272.0,
+      "reward": 0.30666957050561905,
+      "reward_std": 0.37889517843723297,
+      "rewards/concensus_correctness_reward_func/mean": 0.1197500005364418,
+      "rewards/concensus_correctness_reward_func/std": 0.3387041389942169,
+      "rewards/consensus_reward_func/mean": 0.125,
+      "rewards/consensus_reward_func/std": 0.3535533845424652,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.015352241694927216,
-      "rewards/question_recreation_reward_func/std": 0.006488756742328405,
+      "rewards/question_recreation_reward_func/mean": 0.061919582076370716,
+      "rewards/question_recreation_reward_func/std": 0.04408737272024155,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -255,22 +255,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 991.0,
-      "completions/max_terminated_length": 756.5,
-      "completions/mean_length": 509.4375,
-      "completions/mean_terminated_length": 421.9583435058594,
-      "completions/min_length": 36.5,
-      "completions/min_terminated_length": 36.5,
-      "epoch": 0.7088607594936709,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 881.0,
+      "completions/max_terminated_length": 881.0,
+      "completions/mean_length": 285.3125,
+      "completions/mean_terminated_length": 285.3125,
+      "completions/min_length": 49.0,
+      "completions/min_terminated_length": 49.0,
+      "epoch": 0.56,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 4.25427770614624,
-      "kl": 0.001760676721460186,
+      "grad_norm": 7.043668270111084,
+      "kl": 0.002082884529954754,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.2682,
-      "num_tokens": 71347.0,
-      "reward": -0.05695461109280586,
-      "reward_std": 0.22896763868629932,
+      "loss": 0.35,
+      "num_tokens": 59933.0,
+      "reward": 0.06385299749672413,
+      "reward_std": 0.03737428830936551,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -279,14 +279,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.05810788832604885,
-      "rewards/question_recreation_reward_func/std": 0.04673689045011997,
+      "rewards/question_recreation_reward_func/mean": 0.06385299749672413,
+      "rewards/question_recreation_reward_func/std": 0.03331646043807268,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": -0.11506249755620956,
-      "rewards/xmlcount_reward_func/std": 0.3688294589519501,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 14
     },
     {
@@ -295,22 +295,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 214.0,
-      "completions/max_terminated_length": 214.0,
-      "completions/mean_length": 103.6875,
-      "completions/mean_terminated_length": 103.6875,
-      "completions/min_length": 5.0,
-      "completions/min_terminated_length": 5.0,
-      "epoch": 0.810126582278481,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 853.5,
+      "completions/max_terminated_length": 685.0,
+      "completions/mean_length": 341.3125,
+      "completions/mean_terminated_length": 246.9166717529297,
+      "completions/min_length": 64.5,
+      "completions/min_terminated_length": 64.5,
+      "epoch": 0.64,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 12.23747444152832,
-      "kl": 0.007723030605120584,
+      "grad_norm": 5.556763172149658,
+      "kl": 0.0018842843419406563,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0009,
-      "num_tokens": 77102.0,
-      "reward": 0.17531824856996536,
-      "reward_std": 0.03592286352068186,
+      "loss": 0.0971,
+      "num_tokens": 69490.0,
+      "reward": 0.06051425402984023,
+      "reward_std": 0.019080545171163976,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -319,8 +319,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.17531824856996536,
-      "rewards/question_recreation_reward_func/std": 0.045727355405688286,
+      "rewards/question_recreation_reward_func/mean": 0.06051425402984023,
+      "rewards/question_recreation_reward_func/std": 0.030638275435194373,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -335,32 +335,32 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 824.0,
-      "completions/max_terminated_length": 530.5,
-      "completions/mean_length": 194.3125,
-      "completions/mean_terminated_length": 137.34821701049805,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 509.5,
+      "completions/max_terminated_length": 509.5,
+      "completions/mean_length": 192.875,
+      "completions/mean_terminated_length": 192.875,
       "completions/min_length": 3.5,
       "completions/min_terminated_length": 3.5,
-      "epoch": 0.9113924050632911,
+      "epoch": 0.72,
       "frac_reward_zero_std": 0.125,
-      "grad_norm": 9.846582412719727,
-      "kl": 0.006351641248329543,
+      "grad_norm": 6.232974529266357,
+      "kl": 0.006690542242722586,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0552,
-      "num_tokens": 84307.0,
-      "reward": 0.13931425474584103,
-      "reward_std": 0.058048633858561516,
-      "rewards/concensus_correctness_reward_func/mean": 0.0,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "loss": -0.0781,
+      "num_tokens": 76672.0,
+      "reward": 2.7677047792822123,
+      "reward_std": 0.006125873536802828,
+      "rewards/concensus_correctness_reward_func/mean": 2.5,
+      "rewards/concensus_correctness_reward_func/std": 4.629100322723389,
+      "rewards/consensus_reward_func/mean": 0.25,
+      "rewards/consensus_reward_func/std": 0.4629100561141968,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.13931425474584103,
-      "rewards/question_recreation_reward_func/std": 0.15117722004652023,
+      "rewards/question_recreation_reward_func/mean": 0.01770483050495386,
+      "rewards/question_recreation_reward_func/std": 0.01125000836327672,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -375,46 +375,53 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.5,
-      "completions/max_length": 1024.0,
-      "completions/max_terminated_length": 565.0,
-      "completions/mean_length": 665.25,
-      "completions/mean_terminated_length": 306.5,
-      "completions/min_length": 5.0,
-      "completions/min_terminated_length": 5.0,
-      "epoch": 0.9620253164556962,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 564.5,
+      "completions/max_terminated_length": 564.5,
+      "completions/mean_length": 184.0625,
+      "completions/mean_terminated_length": 184.0625,
+      "completions/min_length": 11.5,
+      "completions/min_terminated_length": 11.5,
+      "epoch": 0.8,
       "frac_reward_zero_std": 0.0,
-      "kl": 0.002655731455888599,
-      "num_tokens": 91677.0,
-      "reward": 0.012782756239175797,
-      "reward_std": 0.005160772241652012,
+      "grad_norm": 9.507555961608887,
+      "kl": 0.002568137046182528,
+      "learning_rate": 3.4096741493194193e-09,
+      "loss": 0.0613,
+      "num_tokens": 83713.0,
+      "reward": 0.14513505343347788,
+      "reward_std": 0.180526792537421,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.125,
+      "rewards/consensus_reward_func/std": 0.3535533845424652,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.012782756239175797,
-      "rewards/question_recreation_reward_func/std": 0.008415077812969685,
+      "rewards/question_recreation_reward_func/mean": 0.02013504970818758,
+      "rewards/question_recreation_reward_func/std": 0.011476744432002306,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
       "rewards/xmlcount_reward_func/mean": 0.0,
       "rewards/xmlcount_reward_func/std": 0.0,
-      "step": 19,
+      "step": 20
+    },
+    {
+      "epoch": 0.8,
+      "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.15644574165344238,
-      "train_runtime": 1523.8731,
-      "train_samples_per_second": 0.105,
+      "train_loss": 0.0773889608681202,
+      "train_runtime": 1482.6063,
+      "train_samples_per_second": 0.108,
       "train_steps_per_second": 0.013
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
-  "num_input_tokens_seen": 91677,
+  "num_input_tokens_seen": 83713,
   "num_train_epochs": 1,
   "save_steps": 25,
   "stateful_callbacks": {
@@ -423,8 +430,8 @@
         "should_epoch_stop": false,
         "should_evaluate": false,
         "should_log": false,
-        "should_save": false,
-        "should_training_stop": false
+        "should_save": true,
+        "should_training_stop": true
       },
       "attributes": {}
     }