End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.023168563062790783,
-    "train_runtime": 162.3949,
+    "train_loss": 4.693535439635161,
+    "train_runtime": 167.0439,
     "train_samples": 160,
-    "train_samples_per_second": 1.971,
-    "train_steps_per_second": 0.123
+    "train_samples_per_second": 1.916,
+    "train_steps_per_second": 0.12
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65e40b1e170e5884f7ce696b94e053e444673a92bd46f46f9694e78be9646f1c
+oid sha256:a48784e434429cc978157f1307d0a88d5da208f024eff11f3339fd7b0f189dff
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.023168563062790783,
-    "train_runtime": 162.3949,
+    "train_loss": 4.693535439635161,
+    "train_runtime": 167.0439,
     "train_samples": 160,
-    "train_samples_per_second": 1.971,
-    "train_steps_per_second": 0.123
+    "train_samples_per_second": 1.916,
+    "train_steps_per_second": 0.12
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 231.96875,
+      "completion_length": 227.53125,
       "epoch": 0.1,
-      "grad_norm": 17.455345153808594,
+      "grad_norm": 30.49525260925293,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": -0.0,
-      "reward": 3.6413067949470133,
-      "reward_std": 0.25616460473975167,
-      "rewards/concensus_correctness_reward_func": 0.9591249972581863,
+      "reward": 4.441966446116567,
+      "reward_std": 1.1807270474964753,
+      "rewards/concensus_correctness_reward_func": 1.641249991953373,
       "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6449005694594234,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6907477553468198,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 0.818531246855855,
+      "rewards/strict_format_reward_func": 0.171875,
+      "rewards/xmlcount_reward_func": 0.8755937553360127,
       "step": 2
     },
     {
-      "completion_length": 151.78125,
+      "completion_length": 144.65625,
       "epoch": 0.2,
-      "grad_norm": 89.01654815673828,
-      "kl": 9.351607037766371,
+      "grad_norm": 41.56974792480469,
+      "kl": 2.6795606416417286,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0094,
-      "reward": 6.558691561222076,
-      "reward_std": 0.49686151184141636,
-      "rewards/concensus_correctness_reward_func": 1.9132499694824219,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 0.0027,
+      "reward": 6.809520989656448,
+      "reward_std": 0.14137882058275864,
+      "rewards/concensus_correctness_reward_func": 1.9741249680519104,
+      "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9256291314959526,
+      "rewards/question_recreation_reward_func": 0.9491771869361401,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.1573125012218952,
+      "rewards/strict_format_reward_func": 0.40625,
+      "rewards/xmlcount_reward_func": 1.229968748986721,
       "step": 4
     },
     {
-      "completion_length": 156.75,
+      "completion_length": 164.90625,
       "epoch": 0.3,
-      "grad_norm": 99.52774047851562,
-      "kl": 12.095086781773716,
+      "grad_norm": 58.901611328125,
+      "kl": 3212.812270720955,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0121,
-      "reward": 6.394592821598053,
-      "reward_std": 0.3748498714994639,
-      "rewards/concensus_correctness_reward_func": 1.8473749831318855,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 3.2128,
+      "reward": 6.152694694697857,
+      "reward_std": 0.2201200993731618,
+      "rewards/concensus_correctness_reward_func": 1.727374978363514,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8639678694307804,
+      "rewards/question_recreation_reward_func": 0.9292259570211172,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.1988750025629997,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.16796875,
       "step": 6
     },
     {
-      "completion_length": 150.59375,
+      "completion_length": 156.0625,
       "epoch": 0.4,
-      "grad_norm": 1024.6468505859375,
-      "kl": 69.22306694719009,
+      "grad_norm": 21159.541015625,
+      "kl": 793.1982030207291,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0692,
-      "reward": 6.349600732326508,
-      "reward_std": 0.05679485874134116,
-      "rewards/concensus_correctness_reward_func": 1.8117499789223075,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.7932,
+      "reward": 6.348659709095955,
+      "reward_std": 0.6525374334305525,
+      "rewards/concensus_correctness_reward_func": 1.9973749844357371,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9996007159352303,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8731908686459064,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.2257499992847443,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.1187187507748604,
       "step": 8
     },
     {
-      "completion_length": 141.625,
+      "completion_length": 166.71875,
       "epoch": 0.5,
-      "grad_norm": 17.491762161254883,
-      "kl": 5.087886331835762,
+      "grad_norm": 3487.975830078125,
+      "kl": 235.72644520667382,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0051,
-      "reward": 6.57209238409996,
-      "reward_std": 0.38776873238384724,
-      "rewards/concensus_correctness_reward_func": 2.1048749946057796,
+      "loss": 0.2357,
+      "reward": 6.8310661762952805,
+      "reward_std": 0.966264896094799,
+      "rewards/concensus_correctness_reward_func": 2.3566250018775463,
       "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.881279855966568,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.8903785869479179,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.2109375,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.0059375017881393,
       "step": 10
     },
     {
-      "completion_length": 153.34375,
+      "completion_length": 149.9375,
       "epoch": 0.6,
-      "grad_norm": 45.857887268066406,
-      "kl": 2.472950931522064,
+      "grad_norm": 1146.8641357421875,
+      "kl": 125.67277173465118,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0025,
-      "reward": 6.45940625667572,
-      "reward_std": 0.15495141461724415,
-      "rewards/concensus_correctness_reward_func": 1.9582499787211418,
+      "loss": 0.1257,
+      "reward": 6.07975634932518,
+      "reward_std": 0.34126558783464134,
+      "rewards/concensus_correctness_reward_func": 1.8357499688863754,
       "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.852718744892627,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.8660688688978553,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.421875,
-      "rewards/xmlcount_reward_func": 1.2265625,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.1904375031590462,
       "step": 12
     },
     {
-      "completion_length": 135.0,
+      "completion_length": 141.8125,
       "epoch": 0.7,
-      "grad_norm": 372.874755859375,
-      "kl": 10.58743875962682,
+      "grad_norm": 28.320093154907227,
+      "kl": 42091.95313343528,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0106,
-      "reward": 6.902381718158722,
-      "reward_std": 0.14199074916541576,
-      "rewards/concensus_correctness_reward_func": 2.1672499999403954,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 42.092,
+      "reward": 6.489867955446243,
+      "reward_std": 0.31974472710862756,
+      "rewards/concensus_correctness_reward_func": 2.0317499861121178,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.989037998020649,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.9453992135822773,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.21484375,
+      "rewards/strict_format_reward_func": 0.375,
+      "rewards/xmlcount_reward_func": 1.2002187445759773,
       "step": 14
     },
     {
-      "completion_length": 143.375,
+      "completion_length": 150.53125,
       "epoch": 0.8,
-      "grad_norm": 13.885821342468262,
-      "kl": 0.5841985613078577,
+      "grad_norm": 46.18184280395508,
+      "kl": 1.4570958691183478,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0006,
-      "reward": 6.79749721288681,
-      "reward_std": 0.06372165028005838,
-      "rewards/concensus_correctness_reward_func": 1.9677499905228615,
+      "loss": 0.0015,
+      "reward": 6.744259178638458,
+      "reward_std": 0.062466773204505444,
+      "rewards/concensus_correctness_reward_func": 1.9677499830722809,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9941847957670689,
+      "rewards/question_recreation_reward_func": 0.9749154075980186,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.2418124973773956,
+      "rewards/strict_format_reward_func": 0.4375,
+      "rewards/xmlcount_reward_func": 1.2390937432646751,
       "step": 16
     },
     {
-      "completion_length": 147.125,
+      "completion_length": 143.65625,
       "epoch": 0.9,
-      "grad_norm": 34.52742385864258,
-      "kl": 121.7039143681759,
+      "grad_norm": 19582.970703125,
+      "kl": 464.659947514534,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.1217,
-      "reward": 6.396754413843155,
-      "reward_std": 0.503950642363634,
-      "rewards/concensus_correctness_reward_func": 1.992999978363514,
+      "loss": 0.4647,
+      "reward": 6.67893061041832,
+      "reward_std": 0.07036678654549178,
+      "rewards/concensus_correctness_reward_func": 2.0833749808371067,
       "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9287856929004192,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.9845244064927101,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.162468746304512,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.189156249165535,
       "step": 18
     },
     {
-      "completion_length": 136.6875,
+      "completion_length": 143.5625,
       "epoch": 1.0,
-      "grad_norm": 20.145456314086914,
-      "kl": 0.5796681670472026,
+      "grad_norm": 28.336759567260742,
+      "kl": 7.188884501068969,
       "learning_rate": 0.0,
-      "loss": 0.0006,
-      "reward": 6.883222311735153,
-      "reward_std": 0.20647703111171722,
-      "rewards/concensus_correctness_reward_func": 2.281125009059906,
+      "loss": 0.0072,
+      "reward": 6.240718752145767,
+      "reward_std": 0.3274476360529661,
+      "rewards/concensus_correctness_reward_func": 1.8999999836087227,
       "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9480973184108734,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.8931874856352806,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.421875,
-      "rewards/xmlcount_reward_func": 1.232124999165535,
+      "rewards/strict_format_reward_func": 0.390625,
+      "rewards/xmlcount_reward_func": 1.181906245648861,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.023168563062790783,
-      "train_runtime": 162.3949,
-      "train_samples_per_second": 1.971,
-      "train_steps_per_second": 0.123
+      "train_loss": 4.693535439635161,
+      "train_runtime": 167.0439,
+      "train_samples_per_second": 1.916,
+      "train_steps_per_second": 0.12
     }
   ],
   "logging_steps": 2,