End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.39239501146949e-06,
-    "train_runtime": 9266.1162,
-    "train_samples": 240,
-    "train_samples_per_second": 0.035,
-    "train_steps_per_second": 0.002
+    "train_loss": 1.9294926278234926e-05,
+    "train_runtime": 6487.5089,
+    "train_samples": 433,
+    "train_samples_per_second": 0.049,
+    "train_steps_per_second": 0.003
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e54da1e0fc7b85acb384168dd7500e69373734f40cfae44bad35cc6673c3e6d7
+oid sha256:6388c5eacd132ff0be5a11df472e7bcb66883452cbf5bb29bd179903e8b5c786
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.39239501146949e-06,
-    "train_runtime": 9266.1162,
-    "train_samples": 240,
-    "train_samples_per_second": 0.035,
-    "train_steps_per_second": 0.002
+    "train_loss": 1.9294926278234926e-05,
+    "train_runtime": 6487.5089,
+    "train_samples": 433,
+    "train_samples_per_second": 0.049,
+    "train_steps_per_second": 0.003
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6666666666666666,
+  "epoch": 0.3695150115473441,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 154.90625,
-      "epoch": 0.06666666666666667,
-      "grad_norm": 20.943695068359375,
+      "completion_length": 255.21875,
+      "epoch": 0.03695150115473441,
+      "grad_norm": 31.060409545898438,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 0.1508401128812693,
-      "reward_std": 0.1164285639533773,
+      "reward": 0.0918922832061071,
+      "reward_std": 0.0586454066560691,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.08046511834254488,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.08404853314277716,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.00787500012665987,
+      "rewards/xmlcount_reward_func": 0.007843750063329935,
       "step": 2
     },
     {
-      "completion_length": 179.53125,
-      "epoch": 0.13333333333333333,
-      "grad_norm": 10.192220687866211,
-      "kl": 0.001363701564287112,
+      "completion_length": 167.53125,
+      "epoch": 0.07390300230946882,
+      "grad_norm": 26.61443519592285,
+      "kl": 0.011971469300078752,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 0.1925237558898516,
-      "reward_std": 0.19270968624186935,
+      "reward": 0.07147989561781287,
+      "reward_std": 0.03372790712455753,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.125,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.06752375530777499,
+      "rewards/question_recreation_reward_func": 0.07147989561781287,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
       "rewards/xmlcount_reward_func": 0.0,
       "step": 4
     },
     {
-      "completion_length": 260.21875,
-      "epoch": 0.2,
-      "grad_norm": 45.88401794433594,
-      "kl": 0.03040316281385458,
+      "completion_length": 128.53125,
+      "epoch": 0.11085450346420324,
+      "grad_norm": 6.459836483001709,
+      "kl": 0.009096669891732745,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 0.12052870221668854,
-      "reward_std": 0.14282836719030456,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.3243015332845971,
+      "reward_std": 0.14603966870345175,
+      "rewards/concensus_correctness_reward_func": 0.015625,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.06977870146511123,
+      "rewards/question_recreation_reward_func": 0.11402027792064473,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.011749999597668648,
+      "rewards/xmlcount_reward_func": -0.05534374713897705,
       "step": 6
     },
     {
-      "completion_length": 235.625,
-      "epoch": 0.26666666666666666,
-      "grad_norm": 8.591076850891113,
-      "kl": 0.003081521227613848,
+      "completion_length": 228.125,
+      "epoch": 0.14780600461893764,
+      "grad_norm": 25.84459686279297,
+      "kl": 0.013657524408927202,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 0.04862551300902851,
-      "reward_std": 0.027604442884694436,
+      "reward": 0.14314589684363455,
+      "reward_std": 0.12924392370405258,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.04862551300902851,
+      "rewards/question_recreation_reward_func": 0.08064589614514261,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
       "rewards/xmlcount_reward_func": 0.0,
       "step": 8
     },
     {
-      "completion_length": 193.65625,
-      "epoch": 0.3333333333333333,
-      "grad_norm": 22.166669845581055,
-      "kl": 0.004064998872308934,
+      "completion_length": 160.09375,
+      "epoch": 0.18475750577367206,
+      "grad_norm": 8.251204490661621,
+      "kl": 0.005098440868096077,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 0.10763213399332017,
-      "reward_std": 0.040325247748114634,
+      "reward": 0.20410994556732476,
+      "reward_std": 0.029300687339855358,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.10763213399332017,
+      "rewards/question_recreation_reward_func": 0.07910994556732476,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
       "rewards/xmlcount_reward_func": 0.0,
       "step": 10
     },
     {
-      "completion_length": 154.125,
-      "epoch": 0.4,
-      "grad_norm": 24.712656021118164,
-      "kl": 0.003938008292607265,
+      "completion_length": 109.71875,
+      "epoch": 0.22170900692840648,
+      "grad_norm": 14.531818389892578,
+      "kl": 0.08943804289447144,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0,
-      "reward": 0.07875883619999513,
-      "reward_std": 0.022805250046985748,
+      "loss": 0.0001,
+      "reward": 0.20868571219034493,
+      "reward_std": 0.0271528098055569,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.07875883619999513,
+      "rewards/question_recreation_reward_func": 0.07546696090139449,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.008218750357627869,
       "step": 12
     },
     {
-      "completion_length": 312.5,
-      "epoch": 0.4666666666666667,
-      "grad_norm": 30.27935028076172,
-      "kl": 0.012665021607972449,
+      "completion_length": 124.96875,
+      "epoch": 0.2586605080831409,
+      "grad_norm": 23.98054313659668,
+      "kl": 0.020601566080586053,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 0.021240834670607,
-      "reward_std": 0.08132541776831204,
+      "reward": 0.13461814331822097,
+      "reward_std": 0.11788434895424871,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.058178336126729846,
+      "rewards/question_recreation_reward_func": 0.07211814518086612,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.03693750128149986,
+      "rewards/xmlcount_reward_func": 0.0,
       "step": 14
     },
     {
-      "completion_length": 289.28125,
-      "epoch": 0.5333333333333333,
-      "grad_norm": 22.422969818115234,
-      "kl": 0.014686547433029773,
+      "completion_length": 140.84375,
+      "epoch": 0.2956120092378753,
+      "grad_norm": 25.984272003173828,
+      "kl": 0.012804456360754557,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 0.2178522745380178,
-      "reward_std": 0.20065530513238627,
+      "reward": 0.052462655876297504,
+      "reward_std": 0.04967017705348553,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.092852269182913,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.060150156321469694,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.007687500212341547,
       "step": 16
     },
     {
-      "completion_length": 217.375,
-      "epoch": 0.6,
-      "grad_norm": 25.915138244628906,
-      "kl": 0.010246939140415634,
+      "completion_length": 173.8125,
+      "epoch": 0.3325635103926097,
+      "grad_norm": 9.2888822555542,
+      "kl": 0.011937878574826755,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 0.25968264864059165,
-      "reward_std": 0.11816908442415297,
+      "reward": 0.08258980908431113,
+      "reward_std": 0.03459403937449679,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.1875,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.06602639047196135,
+      "rewards/question_recreation_reward_func": 0.08399605914019048,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.006156249903142452,
+      "rewards/xmlcount_reward_func": -0.0014062500558793545,
       "step": 18
     },
     {
-      "completion_length": 235.9375,
-      "epoch": 0.6666666666666666,
-      "grad_norm": 6.571873188018799,
-      "kl": 0.0036373603479660233,
+      "completion_length": 170.15625,
+      "epoch": 0.3695150115473441,
+      "grad_norm": 16.976003646850586,
+      "kl": 0.018439749983372167,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 0.09592389341560192,
-      "reward_std": 0.05722809254893946,
+      "reward": 0.1647110579360742,
+      "reward_std": 0.0906551209918689,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.09592389341560192,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.11492980879847892,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.012718750163912773,
       "step": 20
     },
     {
-      "epoch": 0.6666666666666666,
+      "epoch": 0.3695150115473441,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 8.39239501146949e-06,
-      "train_runtime": 9266.1162,
-      "train_samples_per_second": 0.035,
-      "train_steps_per_second": 0.002
+      "train_loss": 1.9294926278234926e-05,
+      "train_runtime": 6487.5089,
+      "train_samples_per_second": 0.049,
+      "train_steps_per_second": 0.003
     }
   ],
   "logging_steps": 2,