End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 12919.079583139066,
-    "train_runtime": 146.8015,
+    "train_loss": 34.63017091080546,
+    "train_runtime": 149.3925,
     "train_samples": 160,
-    "train_samples_per_second": 2.18,
-    "train_steps_per_second": 0.136
+    "train_samples_per_second": 2.142,
+    "train_steps_per_second": 0.134
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e8b2a348ae3054fc9f7c99ef7663ff0e79c2889f267743d65e63a13c912d76d
+oid sha256:dc07ae4a8fbf0c1414e320168f24997972793a8b2fedef346b971fadc159d16e
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 12919.079583139066,
-    "train_runtime": 146.8015,
+    "train_loss": 34.63017091080546,
+    "train_runtime": 149.3925,
     "train_samples": 160,
-    "train_samples_per_second": 2.18,
-    "train_steps_per_second": 0.136
+    "train_samples_per_second": 2.142,
+    "train_steps_per_second": 0.134
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 237.71875,
+      "completion_length": 205.625,
       "epoch": 0.1,
-      "grad_norm": 108.31290435791016,
+      "grad_norm": 99.96902465820312,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
-      "loss": 0.0,
-      "reward": 3.658208438195288,
-      "reward_std": 0.866681712330319,
-      "rewards/concensus_correctness_reward_func": 1.0885000005364418,
-      "rewards/consensus_reward_func": 0.875,
+      "loss": -0.0,
+      "reward": 4.236496409517713,
+      "reward_std": 1.159384369617328,
+      "rewards/concensus_correctness_reward_func": 1.258937492966652,
+      "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.608583465218544,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.109375,
-      "rewards/xmlcount_reward_func": 0.7111250013113022,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.6489027393981814,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.140625,
+      "rewards/xmlcount_reward_func": 0.6255312513094395,
       "step": 2
     },
     {
-      "completion_length": 120.0,
+      "completion_length": 143.25,
       "epoch": 0.2,
-      "grad_norm": 729.8348999023438,
-      "kl": 351249.780054101,
+      "grad_norm": 484.384521484375,
+      "kl": 36.32083459291607,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 351.2498,
-      "reward": 5.564554661512375,
-      "reward_std": 0.5053713322704425,
-      "rewards/concensus_correctness_reward_func": 1.6209999918937683,
+      "loss": 0.0363,
+      "reward": 5.732543356716633,
+      "reward_std": 0.5354217339772731,
+      "rewards/concensus_correctness_reward_func": 1.6823749765753746,
       "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8277422818355262,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8452621176838875,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.1001875028014183,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.0799062475562096,
       "step": 4
     },
     {
-      "completion_length": 155.1875,
+      "completion_length": 155.875,
       "epoch": 0.3,
-      "grad_norm": 357.931396484375,
-      "kl": 100.04269157012459,
+      "grad_norm": 169.1754913330078,
+      "kl": 67.9221153233666,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.1,
-      "reward": 5.422056145966053,
-      "reward_std": 0.5586985282134265,
-      "rewards/concensus_correctness_reward_func": 1.5642499811947346,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0679,
+      "reward": 6.084514006972313,
+      "reward_std": 1.0455920902313665,
+      "rewards/concensus_correctness_reward_func": 1.7961874902248383,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8090248834341764,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.8235765150748193,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.0487812478095293,
+      "rewards/xmlcount_reward_func": 1.089749999344349,
       "step": 6
     },
     {
-      "completion_length": 128.875,
+      "completion_length": 136.0625,
       "epoch": 0.4,
-      "grad_norm": 71.76409149169922,
-      "kl": 4.430356410332024,
+      "grad_norm": 203.0240478515625,
+      "kl": 148.1647337176837,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0044,
-      "reward": 5.271592207252979,
-      "reward_std": 1.0291041005111765,
-      "rewards/concensus_correctness_reward_func": 1.5662499889731407,
-      "rewards/consensus_reward_func": 1.4375,
+      "loss": 0.1482,
+      "reward": 5.774978786706924,
+      "reward_std": 0.8584180986508727,
+      "rewards/concensus_correctness_reward_func": 1.6950000068172812,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7706546385306865,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8123538055224344,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 0.9971874989569187,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.0801249966025352,
       "step": 8
     },
     {
-      "completion_length": 120.375,
+      "completion_length": 148.25,
       "epoch": 0.5,
-      "grad_norm": 66.47998809814453,
-      "kl": 4.158945650910027,
+      "grad_norm": 22671.439453125,
+      "kl": 1653.8566977125593,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0042,
-      "reward": 6.34473218023777,
-      "reward_std": 1.0234239008277655,
-      "rewards/concensus_correctness_reward_func": 1.9266249909996986,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 1.6539,
+      "reward": 6.164374992251396,
+      "reward_std": 1.5881682708859444,
+      "rewards/concensus_correctness_reward_func": 1.933499988168478,
+      "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.8893883638083935,
+      "rewards/question_recreation_reward_func": 0.8398749940097332,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.122468750923872,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.078499998897314,
       "step": 10
     },
     {
-      "completion_length": 121.4375,
+      "completion_length": 138.3125,
       "epoch": 0.6,
-      "grad_norm": 158.16209411621094,
-      "kl": 31.171783425146714,
+      "grad_norm": 17988.767578125,
+      "kl": 1401.7687317871023,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0312,
-      "reward": 5.266956835985184,
-      "reward_std": 0.5923136200290173,
-      "rewards/concensus_correctness_reward_func": 1.453374981880188,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 1.4018,
+      "reward": 4.654909428209066,
+      "reward_std": 1.6939838130492717,
+      "rewards/concensus_correctness_reward_func": 1.2355624809861183,
+      "rewards/consensus_reward_func": 1.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.7670194264501333,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.7417220007628202,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.0309375002980232,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.0057500004768372,
       "step": 12
     },
     {
-      "completion_length": 123.5625,
+      "completion_length": 122.75,
       "epoch": 0.7,
-      "grad_norm": 2219.391845703125,
-      "kl": 96.60203694272786,
+      "grad_norm": 112.23609924316406,
+      "kl": 70.25626161438413,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0966,
-      "reward": 4.926824616268277,
-      "reward_std": 1.1714160166011425,
-      "rewards/concensus_correctness_reward_func": 1.5028125159442425,
-      "rewards/consensus_reward_func": 1.1875,
+      "loss": 0.0703,
+      "reward": 5.5448384545743465,
+      "reward_std": 0.9999404510381282,
+      "rewards/concensus_correctness_reward_func": 1.8659375123679638,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8125746981240809,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.750369711255189,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.0489374957978725,
+      "rewards/xmlcount_reward_func": 0.928531251847744,
       "step": 14
     },
     {
-      "completion_length": 128.3125,
+      "completion_length": 134.4375,
       "epoch": 0.8,
-      "grad_norm": 245.07550048828125,
-      "kl": 44.37036015372723,
+      "grad_norm": 18387000.0,
+      "kl": 342814.56765106344,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0444,
-      "reward": 5.480458214879036,
-      "reward_std": 1.5318546425551176,
-      "rewards/concensus_correctness_reward_func": 1.6059374958276749,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 342.8146,
+      "reward": 6.2455944791436195,
+      "reward_std": 0.6316794383637898,
+      "rewards/concensus_correctness_reward_func": 1.9734999909996986,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8120207684114575,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.8281257301568985,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 1.03125,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 1.0533437505364418,
       "step": 16
     },
     {
-      "completion_length": 112.0625,
+      "completion_length": 121.125,
       "epoch": 0.9,
-      "grad_norm": 237.61509704589844,
-      "kl": 15.235337220830843,
+      "grad_norm": 5167.2353515625,
+      "kl": 102.93470847699791,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0152,
-      "reward": 5.376467615365982,
-      "reward_std": 1.1847553994884947,
-      "rewards/concensus_correctness_reward_func": 1.7651874907314777,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.1029,
+      "reward": 5.336302071809769,
+      "reward_std": 1.0969595974311233,
+      "rewards/concensus_correctness_reward_func": 1.5333124920725822,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8065925846749451,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.8231771271675825,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.1328125,
+      "rewards/strict_format_reward_func": 0.28125,
+      "rewards/xmlcount_reward_func": 1.0735625009983778,
       "step": 18
     },
     {
-      "completion_length": 107.8125,
+      "completion_length": 124.8125,
       "epoch": 1.0,
-      "grad_norm": 492.7001647949219,
-      "kl": 128839242.43654859,
+      "grad_norm": 82.51132202148438,
+      "kl": 5.90467467578128,
       "learning_rate": 0.0,
-      "loss": 128839.25,
-      "reward": 5.459323972463608,
-      "reward_std": 0.6939402413454445,
-      "rewards/concensus_correctness_reward_func": 1.6924999952316284,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0059,
+      "reward": 5.849171329289675,
+      "reward_std": 0.5187715581487282,
+      "rewards/concensus_correctness_reward_func": 1.7366249822080135,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.688698960468173,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.8312963852658868,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.03125,
+      "rewards/xmlcount_reward_func": 1.171875,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 12919.079583139066,
-      "train_runtime": 146.8015,
-      "train_samples_per_second": 2.18,
-      "train_steps_per_second": 0.136
+      "train_loss": 34.63017091080546,
+      "train_runtime": 149.3925,
+      "train_samples_per_second": 2.142,
+      "train_steps_per_second": 0.134
     }
   ],
   "logging_steps": 2,