End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.63293201594206e-06,
-    "train_runtime": 518.9769,
+    "train_loss": 6.427429616451263e-06,
+    "train_runtime": 417.5088,
     "train_samples": 160,
-    "train_samples_per_second": 0.617,
-    "train_steps_per_second": 0.039
+    "train_samples_per_second": 0.766,
+    "train_steps_per_second": 0.048
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29149bcad4ce96fb58ca4ac47b9367cc50b1300616f8a3f8cc163445c54f07b7
+oid sha256:dbc5afc19758d295579dfa561a5788ac45a7b8d4f33a6004b538ee02f88170f4
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.63293201594206e-06,
-    "train_runtime": 518.9769,
+    "train_loss": 6.427429616451263e-06,
+    "train_runtime": 417.5088,
     "train_samples": 160,
-    "train_samples_per_second": 0.617,
-    "train_steps_per_second": 0.039
+    "train_samples_per_second": 0.766,
+    "train_steps_per_second": 0.048
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 390.84375,
+      "completion_length": 391.84375,
       "epoch": 0.1,
-      "grad_norm": 5.927030563354492,
+      "grad_norm": 4.776199817657471,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 1.0162433590739965,
-      "reward_std": 0.7680135749978945,
-      "rewards/concensus_correctness_reward_func": 0.1028124988079071,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.6934495995519683,
+      "reward_std": 0.8525555054657161,
+      "rewards/concensus_correctness_reward_func": 0.08275000005960464,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.4383371048606932,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.4939496314327698,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.28759374283254147,
+      "rewards/xmlcount_reward_func": 0.49174999515525997,
       "step": 2
     },
     {
-      "completion_length": 486.40625,
+      "completion_length": 395.4375,
       "epoch": 0.2,
-      "grad_norm": 11.698854446411133,
-      "kl": 0.0013407280239334796,
+      "grad_norm": 9.231148719787598,
+      "kl": 0.0014984993867983576,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 2.1477307975292206,
-      "reward_std": 1.1042591950390488,
-      "rewards/concensus_correctness_reward_func": 0.25625000055879354,
+      "reward": 2.3257811442017555,
+      "reward_std": 1.0566792025929317,
+      "rewards/concensus_correctness_reward_func": 0.35550000332295895,
       "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.4915745386388153,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5732811409980059,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6499062292277813,
+      "rewards/xmlcount_reward_func": 0.7719999738037586,
       "step": 4
     },
     {
-      "completion_length": 567.65625,
+      "completion_length": 420.0625,
       "epoch": 0.3,
-      "grad_norm": 2.725572109222412,
-      "kl": 0.001858470874140039,
+      "grad_norm": 20.898244857788086,
+      "kl": 0.0021197714013396762,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 1.5479721883311868,
-      "reward_std": 0.7916838391683996,
-      "rewards/concensus_correctness_reward_func": 0.0078125,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.476895283907652,
+      "reward_std": 1.1881672106683254,
+      "rewards/concensus_correctness_reward_func": 0.06462499871850014,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5597534505650401,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5145827787928283,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6054062452167273,
+      "rewards/xmlcount_reward_func": 0.5226874835789204,
       "step": 6
     },
     {
-      "completion_length": 461.96875,
+      "completion_length": 505.6875,
       "epoch": 0.4,
-      "grad_norm": 4.154588222503662,
-      "kl": 0.003220928425434977,
+      "grad_norm": 4.914966583251953,
+      "kl": 0.005482891705469228,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 1.941092213615775,
-      "reward_std": 1.3058097834000364,
-      "rewards/concensus_correctness_reward_func": 0.19812500104308128,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 2.152218254748732,
+      "reward_std": 1.4283872889354825,
+      "rewards/concensus_correctness_reward_func": 0.49387499690055847,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5833109403029084,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5518744704313576,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.47215622290968895,
+      "rewards/xmlcount_reward_func": 0.48146875761449337,
       "step": 8
     },
     {
-      "completion_length": 571.78125,
+      "completion_length": 425.21875,
       "epoch": 0.5,
-      "grad_norm": 4.0821213722229,
-      "kl": 0.0037878142975387163,
+      "grad_norm": 5.341366767883301,
+      "kl": 0.006879790460516233,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 2.0178673220798373,
-      "reward_std": 1.0821893326065037,
-      "rewards/concensus_correctness_reward_func": 0.07074999995529652,
+      "reward": 2.214655563235283,
+      "reward_std": 1.2052832515910268,
+      "rewards/concensus_correctness_reward_func": 0.10068750008940697,
       "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5175235585775226,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5675930762663484,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.49209374375641346,
+      "rewards/xmlcount_reward_func": 0.796374985948205,
       "step": 10
     },
     {
-      "completion_length": 482.78125,
+      "completion_length": 442.5625,
       "epoch": 0.6,
-      "grad_norm": 4.6886982917785645,
-      "kl": 0.005024600162869319,
+      "grad_norm": 26.609264373779297,
+      "kl": 0.009219564337399788,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 1.8218787871301174,
-      "reward_std": 1.2323249001055956,
-      "rewards/concensus_correctness_reward_func": 0.13056250102818012,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.690632976591587,
+      "reward_std": 1.378328304272145,
+      "rewards/concensus_correctness_reward_func": 0.2481249999254942,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.48856626776978374,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4131954708136618,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7027499992400408,
+      "rewards/xmlcount_reward_func": 0.5293124988675117,
       "step": 12
     },
     {
-      "completion_length": 402.5625,
+      "completion_length": 428.0,
       "epoch": 0.7,
-      "grad_norm": 5.085709095001221,
-      "kl": 0.00840434868587181,
+      "grad_norm": 4.339048862457275,
+      "kl": 0.00990395323606208,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 2.2788943741470575,
-      "reward_std": 1.0931653431616724,
-      "rewards/concensus_correctness_reward_func": 0.11487499810755253,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.9994048587977886,
+      "reward_std": 1.3680745382444002,
+      "rewards/concensus_correctness_reward_func": 0.1614375002682209,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.6328318938612938,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5740923208650202,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.78118746727705,
+      "rewards/xmlcount_reward_func": 0.6388749964535236,
       "step": 14
     },
     {
-      "completion_length": 413.90625,
+      "completion_length": 438.03125,
       "epoch": 0.8,
-      "grad_norm": 4.9335527420043945,
-      "kl": 0.009245744353393093,
+      "grad_norm": 4.747790813446045,
+      "kl": 0.007855257164919749,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 2.3618225418031216,
-      "reward_std": 1.3073595315217972,
-      "rewards/concensus_correctness_reward_func": 0.14831249974668026,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 1.9228089675307274,
+      "reward_std": 0.6404511121800169,
+      "rewards/concensus_correctness_reward_func": 0.11306249909102917,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6038850257173181,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.6690589864738286,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7971249856054783,
+      "rewards/xmlcount_reward_func": 0.7031874842941761,
       "step": 16
     },
     {
-      "completion_length": 501.65625,
+      "completion_length": 416.875,
       "epoch": 0.9,
-      "grad_norm": 12.62442398071289,
-      "kl": 0.007458922351361252,
+      "grad_norm": 5.46245002746582,
+      "kl": 0.008813429856672883,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 1.7577764000743628,
-      "reward_std": 1.32415815704735,
-      "rewards/concensus_correctness_reward_func": 0.09475000016391277,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.7960961200296879,
+      "reward_std": 1.2154418476857245,
+      "rewards/concensus_correctness_reward_func": 0.12325000017881393,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5351514117792249,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.6314085975755006,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5028749844059348,
+      "rewards/xmlcount_reward_func": 0.728937485255301,
       "step": 18
     },
     {
-      "completion_length": 489.6875,
+      "completion_length": 418.15625,
       "epoch": 1.0,
-      "grad_norm": 4.438209533691406,
-      "kl": 0.009675627807155252,
+      "grad_norm": 7.303548812866211,
+      "kl": 0.012376634433167055,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 2.0540846548974514,
-      "reward_std": 0.9402668559923768,
-      "rewards/concensus_correctness_reward_func": 0.41724999248981476,
+      "reward": 2.181873619556427,
+      "reward_std": 1.3524082526564598,
+      "rewards/concensus_correctness_reward_func": 0.18056249991059303,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.6838346607983112,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.6620298526249826,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5154999792575836,
+      "rewards/xmlcount_reward_func": 0.5892812423408031,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 5.63293201594206e-06,
-      "train_runtime": 518.9769,
-      "train_samples_per_second": 0.617,
-      "train_steps_per_second": 0.039
+      "train_loss": 6.427429616451263e-06,
+      "train_runtime": 417.5088,
+      "train_samples_per_second": 0.766,
+      "train_steps_per_second": 0.048
     }
   ],
   "logging_steps": 2,