End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 88.58008198263124,
-    "train_runtime": 103.6296,
+    "train_loss": 20766.248711165834,
+    "train_runtime": 102.6386,
     "train_samples": 28,
-    "train_samples_per_second": 3.088,
-    "train_steps_per_second": 0.193
+    "train_samples_per_second": 3.118,
+    "train_steps_per_second": 0.195
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2fa6c294c05b730840f4af5f3bcb816b1db7ebda8682a84a81334f9508a50f0b
+oid sha256:3319c4514a75c71a9635812cac658b8cdee638c133e5ea4d8b4da940a081af1b
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 88.58008198263124,
-    "train_runtime": 103.6296,
+    "train_loss": 20766.248711165834,
+    "train_runtime": 102.6386,
     "train_samples": 28,
-    "train_samples_per_second": 3.088,
-    "train_steps_per_second": 0.193
+    "train_samples_per_second": 3.118,
+    "train_steps_per_second": 0.195
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 250.375,
+      "completion_length": 246.53125,
       "epoch": 0.5714285714285714,
-      "grad_norm": 56.84841537475586,
+      "grad_norm": 62.44981002807617,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 2.6311574075371027,
-      "reward_std": 0.7637105587637052,
-      "rewards/concensus_correctness_reward_func": 0.4665000010281801,
-      "rewards/consensus_reward_func": 0.8125,
+      "loss": -0.0,
+      "reward": 2.81018253415823,
+      "reward_std": 1.405245827510953,
+      "rewards/concensus_correctness_reward_func": 0.5839375033974648,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5346261922968552,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5905575299984775,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.6300312462262809,
+      "rewards/strict_format_reward_func": 0.09375,
+      "rewards/xmlcount_reward_func": 0.7919374946504831,
       "step": 2
     },
     {
-      "completion_length": 137.625,
+      "completion_length": 126.41666666666667,
       "epoch": 1.0,
-      "grad_norm": 146.0875701904297,
-      "kl": 2.873302392894402,
+      "grad_norm": 36.17685317993164,
+      "kl": 5.282945941512783,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0022,
-      "reward": 3.824417700370153,
-      "reward_std": 1.31266384323438,
-      "rewards/concensus_correctness_reward_func": 1.177999993165334,
-      "rewards/consensus_reward_func": 0.8333333333333334,
+      "loss": 0.004,
+      "reward": 4.323091174165408,
+      "reward_std": 1.6219139602035284,
+      "rewards/concensus_correctness_reward_func": 1.2741666634877522,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.6210010035041099,
+      "rewards/final_correctness_reward_func": 0.5833333333333334,
+      "rewards/question_recreation_reward_func": 0.642924482623736,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.22916666666666666,
-      "rewards/xmlcount_reward_func": 0.8795833364129066,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 0.906000000735124,
       "step": 4
     },
     {
-      "completion_length": 123.78125,
+      "completion_length": 144.71875,
       "epoch": 1.5714285714285714,
-      "grad_norm": 44.83761215209961,
-      "kl": 1.7949985996820033,
+      "grad_norm": 21567.345703125,
+      "kl": 207468117.82215685,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0018,
-      "reward": 4.477854784578085,
-      "reward_std": 0.6243240954063367,
-      "rewards/concensus_correctness_reward_func": 1.3118750005960464,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 207468.1094,
+      "reward": 5.788680654019117,
+      "reward_std": 0.7503121838381048,
+      "rewards/concensus_correctness_reward_func": 1.78218749538064,
+      "rewards/consensus_reward_func": 1.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6987922478001565,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.8119618319906294,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 0.9828125014901161,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.085156250745058,
       "step": 6
     },
     {
-      "completion_length": 127.75,
+      "completion_length": 143.625,
       "epoch": 2.0,
-      "grad_norm": 76.24980163574219,
-      "kl": 18.988028665383656,
+      "grad_norm": 151.89219665527344,
+      "kl": 48.836665123701096,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0142,
-      "reward": 4.308155059814453,
-      "reward_std": 1.7288466555376847,
-      "rewards/concensus_correctness_reward_func": 1.0689166635274887,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "loss": 0.0366,
+      "reward": 5.109239861369133,
+      "reward_std": 0.582389307130749,
+      "rewards/concensus_correctness_reward_func": 1.4567500005165737,
+      "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.7196967353423437,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.7562398855419209,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2916666666666667,
-      "rewards/xmlcount_reward_func": 0.9778749992450079,
+      "rewards/strict_format_reward_func": 0.3333333333333333,
+      "rewards/xmlcount_reward_func": 1.0629166662693024,
       "step": 8
     },
     {
-      "completion_length": 140.3125,
+      "completion_length": 133.9375,
       "epoch": 2.571428571428571,
-      "grad_norm": 288.7049865722656,
-      "kl": 17.717969600809738,
+      "grad_norm": 6474828.5,
+      "kl": 194250.27258606674,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0177,
-      "reward": 3.9995256760157645,
-      "reward_std": 0.8743778481148183,
-      "rewards/concensus_correctness_reward_func": 1.0920624919235706,
+      "loss": 194.2503,
+      "reward": 4.8971958719193935,
+      "reward_std": 0.6572542649810202,
+      "rewards/concensus_correctness_reward_func": 1.4178124964237213,
       "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5601193871116266,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.7073208371875808,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 0.8629687540233135,
+      "rewards/strict_format_reward_func": 0.28125,
+      "rewards/xmlcount_reward_func": 1.053312499076128,
       "step": 10
     },
     {
-      "completion_length": 105.25,
+      "completion_length": 131.79166666666666,
       "epoch": 3.0,
-      "grad_norm": 80085.2890625,
-      "kl": 4494.306384936829,
+      "grad_norm": 63.40531921386719,
+      "kl": 46.34671560302377,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 3.3707,
-      "reward": 3.7779306173324585,
-      "reward_std": 1.0070178862661123,
-      "rewards/concensus_correctness_reward_func": 0.9075833410024643,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0348,
+      "reward": 5.006111880143483,
+      "reward_std": 0.8839096422307193,
+      "rewards/concensus_correctness_reward_func": 1.4060000032186508,
+      "rewards/consensus_reward_func": 1.1666666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6610139695306619,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.7846952006220818,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 0.9593333303928375,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.0029166663686435,
       "step": 12
     },
     {
-      "completion_length": 158.71875,
+      "completion_length": 145.5625,
       "epoch": 3.571428571428571,
-      "grad_norm": 342.165771484375,
-      "kl": 230.84162104106508,
+      "grad_norm": 49.8672981262207,
+      "kl": 9.690117788501084,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.2308,
-      "reward": 4.614273123443127,
-      "reward_std": 1.2838216330856085,
-      "rewards/concensus_correctness_reward_func": 1.2610000036656857,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0097,
+      "reward": 5.000209610909224,
+      "reward_std": 1.2152829460246721,
+      "rewards/concensus_correctness_reward_func": 1.3089375048875809,
+      "rewards/consensus_reward_func": 1.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.7591793118044734,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.7728346046060324,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 0.9847187455743551,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.04343749769032,
       "step": 14
     },
     {
-      "completion_length": 124.875,
+      "completion_length": 120.5,
       "epoch": 4.0,
-      "grad_norm": 3599872.25,
-      "kl": 77221.04993572489,
+      "grad_norm": 20.376087188720703,
+      "kl": 4.444862203362088,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 57.9158,
-      "reward": 4.892165020108223,
-      "reward_std": 0.47424173859568935,
-      "rewards/concensus_correctness_reward_func": 1.3709166596333187,
-      "rewards/consensus_reward_func": 1.4166666666666667,
+      "loss": 0.0033,
+      "reward": 4.583630581696828,
+      "reward_std": 0.6420133346691728,
+      "rewards/concensus_correctness_reward_func": 1.330999990304311,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7545816494772831,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.7358388638434311,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.2916666666666667,
-      "rewards/xmlcount_reward_func": 1.0583333323399227,
+      "rewards/xmlcount_reward_func": 1.0584583307305973,
       "step": 16
     },
     {
-      "completion_length": 130.65625,
+      "completion_length": 122.59375,
       "epoch": 4.571428571428571,
-      "grad_norm": 6415.2236328125,
-      "kl": 179.9282298819162,
+      "grad_norm": 57.4793701171875,
+      "kl": 5.2552297483198345,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.1799,
-      "reward": 4.808562383055687,
-      "reward_std": 0.8287261514924467,
-      "rewards/concensus_correctness_reward_func": 1.2866249941289425,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0052,
+      "reward": 4.917819622904062,
+      "reward_std": 0.8335820628053625,
+      "rewards/concensus_correctness_reward_func": 1.4153125025331974,
+      "rewards/consensus_reward_func": 1.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7774373296415433,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.7026322261663154,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.056999996304512,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 1.0030000023543835,
       "step": 18
     },
     {
-      "completion_length": 174.45833333333334,
+      "completion_length": 156.41666666666666,
       "epoch": 5.0,
-      "grad_norm": 41.836551666259766,
-      "kl": 1098756.4305711815,
+      "grad_norm": 20.131155014038086,
+      "kl": 45.09713823503504,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 824.0676,
-      "reward": 4.947326357165973,
-      "reward_std": 0.6633909946382724,
-      "rewards/concensus_correctness_reward_func": 1.3980833341677983,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0338,
+      "reward": 4.907328523695469,
+      "reward_std": 0.8428607982350513,
+      "rewards/concensus_correctness_reward_func": 1.4174166570107143,
+      "rewards/consensus_reward_func": 1.1666666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8026596220831076,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.7959951267888149,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 0.9965833326180776,
+      "rewards/strict_format_reward_func": 0.2708333333333333,
+      "rewards/xmlcount_reward_func": 1.089749999344349,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 88.58008198263124,
-      "train_runtime": 103.6296,
-      "train_samples_per_second": 3.088,
-      "train_steps_per_second": 0.193
+      "train_loss": 20766.248711165834,
+      "train_runtime": 102.6386,
+      "train_samples_per_second": 3.118,
+      "train_steps_per_second": 0.195
     }
   ],
   "logging_steps": 2,