End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 280.8794353601872,
-    "train_runtime": 110.6161,
+    "train_loss": 1346.7651600663435,
+    "train_runtime": 114.7259,
     "train_samples": 28,
-    "train_samples_per_second": 2.893,
-    "train_steps_per_second": 0.181
+    "train_samples_per_second": 2.789,
+    "train_steps_per_second": 0.174
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7605f574fd5d2e0f59a9c0a64191abeec73adec761a9279b7871a85ce1de76d
+oid sha256:bd4f0d99b73e85ae32a03c5b29a6f6f7cbdc27edfc00f7c42f043dd407c5ebb2
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 280.8794353601872,
-    "train_runtime": 110.6161,
+    "train_loss": 1346.7651600663435,
+    "train_runtime": 114.7259,
     "train_samples": 28,
-    "train_samples_per_second": 2.893,
-    "train_steps_per_second": 0.181
+    "train_samples_per_second": 2.789,
+    "train_steps_per_second": 0.174
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 240.28125,
+      "completion_length": 254.6875,
       "epoch": 0.5714285714285714,
-      "grad_norm": 63.92334747314453,
+      "grad_norm": 49.11457824707031,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 3.2025501457974315,
-      "reward_std": 0.6751712255645543,
-      "rewards/concensus_correctness_reward_func": 0.7447499930858612,
-      "rewards/consensus_reward_func": 0.875,
+      "reward": 3.2836188105866313,
+      "reward_std": 0.7263518713589292,
+      "rewards/concensus_correctness_reward_func": 0.7724999897181988,
+      "rewards/consensus_reward_func": 1.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6419876753352582,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4882438706699759,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.125,
-      "rewards/xmlcount_reward_func": 0.7533125085756183,
+      "rewards/strict_format_reward_func": 0.171875,
+      "rewards/xmlcount_reward_func": 0.7884999979287386,
       "step": 2
     },
     {
-      "completion_length": 184.70833333333334,
+      "completion_length": 164.83333333333334,
       "epoch": 1.0,
-      "grad_norm": 395.1527099609375,
-      "kl": 24.505333456831675,
+      "grad_norm": 79.6348876953125,
+      "kl": 1.8822873709723353,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0184,
-      "reward": 5.200208688775699,
-      "reward_std": 1.142505804542452,
-      "rewards/concensus_correctness_reward_func": 1.6190000077088673,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0014,
+      "reward": 5.652250826358795,
+      "reward_std": 0.6764374932196612,
+      "rewards/concensus_correctness_reward_func": 1.8013333280881245,
+      "rewards/consensus_reward_func": 1.5833333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.7657920600225528,
+      "rewards/question_recreation_reward_func": 0.7254591931899389,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.20833333333333334,
-      "rewards/xmlcount_reward_func": 1.0237499997019768,
+      "rewards/strict_format_reward_func": 0.2916666666666667,
+      "rewards/xmlcount_reward_func": 1.1671250015497208,
       "step": 4
     },
     {
-      "completion_length": 157.75,
+      "completion_length": 169.1875,
       "epoch": 1.5714285714285714,
-      "grad_norm": 5595.49951171875,
-      "kl": 104.04436226189137,
+      "grad_norm": 916771200.0,
+      "kl": 13428740.037678678,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.104,
-      "reward": 5.785204350948334,
-      "reward_std": 1.0568289030343294,
-      "rewards/concensus_correctness_reward_func": 1.7255000062286854,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 13428.7402,
+      "reward": 5.615537449717522,
+      "reward_std": 0.5303303725086153,
+      "rewards/concensus_correctness_reward_func": 1.7261249907314777,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9176418744027615,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.825818732380867,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.1576875038444996,
+      "rewards/xmlcount_reward_func": 1.141718752682209,
       "step": 6
     },
     {
-      "completion_length": 173.125,
+      "completion_length": 166.75,
       "epoch": 2.0,
-      "grad_norm": 87.04039764404297,
-      "kl": 2.6987843898435435,
+      "grad_norm": 42.46098709106445,
+      "kl": 4.602054620782535,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.002,
-      "reward": 4.84399555126826,
-      "reward_std": 0.4288954818621278,
-      "rewards/concensus_correctness_reward_func": 1.152583325902621,
-      "rewards/consensus_reward_func": 1.4166666666666667,
+      "loss": 0.0035,
+      "reward": 5.048217097918193,
+      "reward_std": 0.8972721509635448,
+      "rewards/concensus_correctness_reward_func": 1.2505833208560944,
+      "rewards/consensus_reward_func": 1.6666666666666667,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7874538333465656,
+      "rewards/question_recreation_reward_func": 0.6336337703590592,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3333333333333333,
-      "rewards/xmlcount_reward_func": 1.1539583454529445,
+      "rewards/strict_format_reward_func": 0.3541666666666667,
+      "rewards/xmlcount_reward_func": 1.1431666612625122,
       "step": 8
     },
     {
-      "completion_length": 163.53125,
+      "completion_length": 161.59375,
       "epoch": 2.571428571428571,
-      "grad_norm": 56.5728759765625,
-      "kl": 1.7313345186412334,
+      "grad_norm": 37.895320892333984,
+      "kl": 38800.4430101281,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0017,
-      "reward": 5.237629234790802,
-      "reward_std": 1.271119650453329,
-      "rewards/concensus_correctness_reward_func": 1.508874997496605,
+      "loss": 38.8004,
+      "reward": 5.324525937438011,
+      "reward_std": 0.8662179499806371,
+      "rewards/concensus_correctness_reward_func": 1.4884374886751175,
       "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6908793298061937,
+      "rewards/question_recreation_reward_func": 0.8118072021752596,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.16287499666214,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.1649062596261501,
       "step": 10
     },
     {
-      "completion_length": 153.75,
+      "completion_length": 155.54166666666666,
       "epoch": 3.0,
-      "grad_norm": 64.883544921875,
-      "kl": 3744596.9175575543,
+      "grad_norm": 92.2210922241211,
+      "kl": 11.152309625719985,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 2808.4473,
-      "reward": 5.073945393164952,
-      "reward_std": 0.7713358254016688,
-      "rewards/concensus_correctness_reward_func": 1.3540833344062169,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0084,
+      "reward": 5.643816947937012,
+      "reward_std": 0.5692423766789337,
+      "rewards/concensus_correctness_reward_func": 1.6060833235581715,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7634453773498535,
+      "rewards/question_recreation_reward_func": 0.8004835819204649,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.143916666507721,
+      "rewards/strict_format_reward_func": 0.3541666666666667,
+      "rewards/xmlcount_reward_func": 1.1330833385388057,
       "step": 12
     },
     {
-      "completion_length": 151.28125,
+      "completion_length": 163.46875,
       "epoch": 3.571428571428571,
-      "grad_norm": 54.6231575012207,
-      "kl": 2.972014800645411,
+      "grad_norm": 167.47503662109375,
+      "kl": 13.075925993267447,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.003,
-      "reward": 5.40961492061615,
-      "reward_std": 0.532777672924567,
-      "rewards/concensus_correctness_reward_func": 1.5808750130236149,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0131,
+      "reward": 5.369014501571655,
+      "reward_std": 0.4032270271272864,
+      "rewards/concensus_correctness_reward_func": 1.3764999955892563,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8442399445921183,
+      "rewards/question_recreation_reward_func": 0.896764550358057,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.1095000058412552,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.2207500040531158,
       "step": 14
     },
     {
-      "completion_length": 145.25,
+      "completion_length": 167.125,
       "epoch": 4.0,
-      "grad_norm": 33.24628829956055,
-      "kl": 2.293897901661694,
+      "grad_norm": 154.0272674560547,
+      "kl": 99.95235991602142,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0017,
-      "reward": 6.327785889307658,
-      "reward_std": 0.27025019881936413,
-      "rewards/concensus_correctness_reward_func": 1.9228333135445912,
-      "rewards/consensus_reward_func": 1.8333333333333333,
+      "loss": 0.075,
+      "reward": 4.875289618968964,
+      "reward_std": 1.0041625554828595,
+      "rewards/concensus_correctness_reward_func": 1.3239999810854595,
+      "rewards/consensus_reward_func": 1.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.7730359372993311,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.7283729445189238,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3958333333333333,
-      "rewards/xmlcount_reward_func": 1.2360833287239075,
+      "rewards/strict_format_reward_func": 0.2708333333333333,
+      "rewards/xmlcount_reward_func": 1.0520833333333333,
       "step": 16
     },
     {
-      "completion_length": 143.96875,
+      "completion_length": 157.3125,
       "epoch": 4.571428571428571,
-      "grad_norm": 92.10546875,
-      "kl": 4.8783937285188586,
+      "grad_norm": 43.15923309326172,
+      "kl": 3.150787123478949,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0049,
-      "reward": 5.564976543188095,
-      "reward_std": 0.5592622863623546,
-      "rewards/concensus_correctness_reward_func": 1.5291250012814999,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0032,
+      "reward": 5.6706234365701675,
+      "reward_std": 0.9191514646518044,
+      "rewards/concensus_correctness_reward_func": 1.5298749953508377,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8085703207179904,
+      "rewards/question_recreation_reward_func": 0.7447797094937414,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.1335312500596046,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.1928437426686287,
       "step": 18
     },
     {
-      "completion_length": 168.83333333333334,
+      "completion_length": 176.58333333333334,
       "epoch": 5.0,
-      "grad_norm": 29.19004249572754,
-      "kl": 281.794287091742,
+      "grad_norm": 312.3111877441406,
+      "kl": 8.666269062707821,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.2113,
-      "reward": 5.404958148797353,
-      "reward_std": 0.8559029260262226,
-      "rewards/concensus_correctness_reward_func": 1.5899166613817215,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0065,
+      "reward": 4.999991357326508,
+      "reward_std": 0.8645469406619668,
+      "rewards/concensus_correctness_reward_func": 1.4022499819596608,
+      "rewards/consensus_reward_func": 1.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.7637081438054641,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.7614080148438612,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1555000046888988,
+      "rewards/strict_format_reward_func": 0.3541666666666667,
+      "rewards/xmlcount_reward_func": 1.148833344380061,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 280.8794353601872,
-      "train_runtime": 110.6161,
-      "train_samples_per_second": 2.893,
-      "train_steps_per_second": 0.181
+      "train_loss": 1346.7651600663435,
+      "train_runtime": 114.7259,
+      "train_samples_per_second": 2.789,
+      "train_steps_per_second": 0.174
     }
   ],
   "logging_steps": 2,