End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 34.63017091080546,
-    "train_runtime": 149.3925,
+    "train_loss": 6.360948879132048,
+    "train_runtime": 156.959,
     "train_samples": 160,
-    "train_samples_per_second": 2.142,
-    "train_steps_per_second": 0.134
+    "train_samples_per_second": 2.039,
+    "train_steps_per_second": 0.127
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc07ae4a8fbf0c1414e320168f24997972793a8b2fedef346b971fadc159d16e
+oid sha256:e66757d33a4ef759c72a4e60b1818a0deada2eb3bd3ac7a0577ce18a0ead3b75
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 34.63017091080546,
-    "train_runtime": 149.3925,
+    "train_loss": 6.360948879132048,
+    "train_runtime": 156.959,
     "train_samples": 160,
-    "train_samples_per_second": 2.142,
-    "train_steps_per_second": 0.134
+    "train_samples_per_second": 2.039,
+    "train_steps_per_second": 0.127
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 205.625,
+      "completion_length": 223.9375,
       "epoch": 0.1,
-      "grad_norm": 99.96902465820312,
+      "grad_norm": 203.7086944580078,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": -0.0,
-      "reward": 4.236496409517713,
-      "reward_std": 1.159384369617328,
-      "rewards/concensus_correctness_reward_func": 1.258937492966652,
-      "rewards/consensus_reward_func": 1.25,
+      "reward": 3.9444722017506137,
+      "reward_std": 1.790060242288746,
+      "rewards/concensus_correctness_reward_func": 1.4638749994337559,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.6489027393981814,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6139097640989348,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.140625,
-      "rewards/xmlcount_reward_func": 0.6255312513094395,
+      "rewards/strict_format_reward_func": 0.1875,
+      "rewards/xmlcount_reward_func": 0.741687492467463,
       "step": 2
     },
     {
-      "completion_length": 143.25,
+      "completion_length": 158.875,
       "epoch": 0.2,
-      "grad_norm": 484.384521484375,
-      "kl": 36.32083459291607,
+      "grad_norm": 39.68674850463867,
+      "kl": 1.1318362320307642,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0363,
-      "reward": 5.732543356716633,
-      "reward_std": 0.5354217339772731,
-      "rewards/concensus_correctness_reward_func": 1.6823749765753746,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0011,
+      "reward": 6.102113731205463,
+      "reward_std": 0.8478451119735837,
+      "rewards/concensus_correctness_reward_func": 1.8132499977946281,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8452621176838875,
+      "rewards/question_recreation_reward_func": 0.9185200277715921,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.0799062475562096,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.1359687484800816,
       "step": 4
     },
     {
-      "completion_length": 155.875,
+      "completion_length": 188.875,
       "epoch": 0.3,
-      "grad_norm": 169.1754913330078,
-      "kl": 67.9221153233666,
+      "grad_norm": 1694169.5,
+      "kl": 61022.59591866261,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0679,
-      "reward": 6.084514006972313,
-      "reward_std": 1.0455920902313665,
-      "rewards/concensus_correctness_reward_func": 1.7961874902248383,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 61.0226,
+      "reward": 4.908621296286583,
+      "reward_std": 1.312034587652306,
+      "rewards/concensus_correctness_reward_func": 1.364124983549118,
+      "rewards/consensus_reward_func": 1.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.8235765150748193,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.7794337533414364,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.089749999344349,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 1.0619374997913837,
       "step": 6
     },
     {
-      "completion_length": 136.0625,
+      "completion_length": 151.875,
       "epoch": 0.4,
-      "grad_norm": 203.0240478515625,
-      "kl": 148.1647337176837,
+      "grad_norm": 492.3763122558594,
+      "kl": 210.9650375645142,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.1482,
-      "reward": 5.774978786706924,
-      "reward_std": 0.8584180986508727,
-      "rewards/concensus_correctness_reward_func": 1.6950000068172812,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.211,
+      "reward": 5.924358628690243,
+      "reward_std": 0.5515469368910999,
+      "rewards/concensus_correctness_reward_func": 1.8797499937936664,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8123538055224344,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.8393273764522746,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.0801249966025352,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 1.080281250178814,
       "step": 8
     },
     {
-      "completion_length": 148.25,
+      "completion_length": 133.8125,
       "epoch": 0.5,
-      "grad_norm": 22671.439453125,
-      "kl": 1653.8566977125593,
+      "grad_norm": 24558.71875,
+      "kl": 721.3866671086289,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 1.6539,
-      "reward": 6.164374992251396,
-      "reward_std": 1.5881682708859444,
-      "rewards/concensus_correctness_reward_func": 1.933499988168478,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.7214,
+      "reward": 6.406012073159218,
+      "reward_std": 0.7330344214569777,
+      "rewards/concensus_correctness_reward_func": 2.0498749911785126,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.8398749940097332,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.8526683263480663,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.078499998897314,
+      "rewards/strict_format_reward_func": 0.28125,
+      "rewards/xmlcount_reward_func": 1.0972187519073486,
       "step": 10
     },
     {
-      "completion_length": 138.3125,
+      "completion_length": 135.65625,
       "epoch": 0.6,
-      "grad_norm": 17988.767578125,
-      "kl": 1401.7687317871023,
+      "grad_norm": 64.38353729248047,
+      "kl": 263.8177743591368,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 1.4018,
-      "reward": 4.654909428209066,
-      "reward_std": 1.6939838130492717,
-      "rewards/concensus_correctness_reward_func": 1.2355624809861183,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.2638,
+      "reward": 5.379809558391571,
+      "reward_std": 0.5953573631122708,
+      "rewards/concensus_correctness_reward_func": 1.4174999743700027,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7417220007628202,
+      "rewards/question_recreation_reward_func": 0.8784658461809158,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.0057500004768372,
+      "rewards/strict_format_reward_func": 0.375,
+      "rewards/xmlcount_reward_func": 1.1463437527418137,
       "step": 12
     },
     {
-      "completion_length": 122.75,
+      "completion_length": 162.0,
       "epoch": 0.7,
-      "grad_norm": 112.23609924316406,
-      "kl": 70.25626161438413,
+      "grad_norm": 4812.80224609375,
+      "kl": 210.1550904882606,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0703,
-      "reward": 5.5448384545743465,
-      "reward_std": 0.9999404510381282,
-      "rewards/concensus_correctness_reward_func": 1.8659375123679638,
+      "loss": 0.2102,
+      "reward": 5.389181062579155,
+      "reward_std": 1.1157905644795392,
+      "rewards/concensus_correctness_reward_func": 1.6220000125467777,
       "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.750369711255189,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8566497433930635,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 0.928531251847744,
+      "rewards/strict_format_reward_func": 0.21875,
+      "rewards/xmlcount_reward_func": 1.0042812526226044,
       "step": 14
     },
     {
-      "completion_length": 134.4375,
+      "completion_length": 159.125,
       "epoch": 0.8,
-      "grad_norm": 18387000.0,
-      "kl": 342814.56765106344,
+      "grad_norm": 692.3638305664062,
+      "kl": 134.42648913431913,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 342.8146,
-      "reward": 6.2455944791436195,
-      "reward_std": 0.6316794383637898,
-      "rewards/concensus_correctness_reward_func": 1.9734999909996986,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.1344,
+      "reward": 6.275528252124786,
+      "reward_std": 0.8024359941482544,
+      "rewards/concensus_correctness_reward_func": 1.9732499942183495,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.8281257301568985,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8014658335596323,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.0533437505364418,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.0476875007152557,
       "step": 16
     },
     {
-      "completion_length": 121.125,
+      "completion_length": 123.9375,
       "epoch": 0.9,
-      "grad_norm": 5167.2353515625,
-      "kl": 102.93470847699791,
+      "grad_norm": 817.1598510742188,
+      "kl": 399.1941711329855,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.1029,
-      "reward": 5.336302071809769,
-      "reward_std": 1.0969595974311233,
-      "rewards/concensus_correctness_reward_func": 1.5333124920725822,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.3992,
+      "reward": 4.676305454224348,
+      "reward_std": 0.88509076932678,
+      "rewards/concensus_correctness_reward_func": 1.2936249822378159,
+      "rewards/consensus_reward_func": 1.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.8231771271675825,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.7137429475260433,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.0735625009983778,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 0.9970625024288893,
       "step": 18
     },
     {
-      "completion_length": 124.8125,
+      "completion_length": 147.625,
       "epoch": 1.0,
-      "grad_norm": 82.51132202148438,
-      "kl": 5.90467467578128,
+      "grad_norm": 1609.4205322265625,
+      "kl": 645.8300444511697,
       "learning_rate": 0.0,
-      "loss": 0.0059,
-      "reward": 5.849171329289675,
-      "reward_std": 0.5187715581487282,
-      "rewards/concensus_correctness_reward_func": 1.7366249822080135,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.6458,
+      "reward": 4.875687658786774,
+      "reward_std": 1.68803179403767,
+      "rewards/concensus_correctness_reward_func": 1.3708749897778034,
+      "rewards/consensus_reward_func": 1.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8312963852658868,
+      "rewards/question_recreation_reward_func": 0.8263127245008945,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.171875,
+      "rewards/xmlcount_reward_func": 1.0691250041127205,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 34.63017091080546,
-      "train_runtime": 149.3925,
-      "train_samples_per_second": 2.142,
-      "train_steps_per_second": 0.134
+      "train_loss": 6.360948879132048,
+      "train_runtime": 156.959,
+      "train_samples_per_second": 2.039,
+      "train_steps_per_second": 0.127
     }
   ],
   "logging_steps": 2,