End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 10327424264.741564,
-    "train_runtime": 160.9363,
+    "train_loss": 32604.248746631725,
+    "train_runtime": 165.2741,
     "train_samples": 28,
-    "train_samples_per_second": 1.988,
-    "train_steps_per_second": 0.124
+    "train_samples_per_second": 1.936,
+    "train_steps_per_second": 0.121
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b28ba4e7e238657b146d09855a6bf7c593efc1174e26f65e1fe956f6b8211bb
+oid sha256:6773a4747965d8b2ecf2c196846a8002624fa6055a9e5fd95a42e0c38ae7dfd3
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 10327424264.741564,
-    "train_runtime": 160.9363,
+    "train_loss": 32604.248746631725,
+    "train_runtime": 165.2741,
     "train_samples": 28,
-    "train_samples_per_second": 1.988,
-    "train_steps_per_second": 0.124
+    "train_samples_per_second": 1.936,
+    "train_steps_per_second": 0.121
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 250.40625,
+      "completion_length": 270.125,
       "epoch": 0.5714285714285714,
-      "grad_norm": 96.9992904663086,
+      "grad_norm": 32.114776611328125,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 2.7266246154904366,
-      "reward_std": 0.7908698467799695,
-      "rewards/concensus_correctness_reward_func": 0.5036249980330467,
-      "rewards/consensus_reward_func": 0.5625,
+      "loss": 0.0,
+      "reward": 3.6483192779123783,
+      "reward_std": 1.990112421743106,
+      "rewards/concensus_correctness_reward_func": 1.1149374954402447,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7141870809718966,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.712194187566638,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.9306874834001064,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.8211874882690609,
       "step": 2
     },
     {
-      "completion_length": 211.125,
+      "completion_length": 171.83333333333334,
       "epoch": 1.0,
-      "grad_norm": 596.4487915039062,
-      "kl": 50.37280273879878,
+      "grad_norm": 23.08681297302246,
+      "kl": 1.149132039397955,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0378,
-      "reward": 5.132219831148784,
-      "reward_std": 1.542186655589224,
-      "rewards/concensus_correctness_reward_func": 1.6507500112056732,
-      "rewards/consensus_reward_func": 1.0833333333333333,
+      "loss": 0.0009,
+      "reward": 4.261914700269699,
+      "reward_std": 0.24328408813259253,
+      "rewards/concensus_correctness_reward_func": 1.1205000181992848,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.8855114231506983,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.9252896656592687,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0959582924842834,
+      "rewards/xmlcount_reward_func": 1.0494583000739415,
       "step": 4
     },
     {
-      "completion_length": 178.03125,
+      "completion_length": 186.0,
       "epoch": 1.5714285714285714,
-      "grad_norm": 676.3795166015625,
-      "kl": 43.27882198570296,
+      "grad_norm": 8824.3994140625,
+      "kl": 325670162.8957561,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0433,
-      "reward": 4.546325594186783,
-      "reward_std": 1.3957796231843531,
-      "rewards/concensus_correctness_reward_func": 1.425625003874302,
-      "rewards/consensus_reward_func": 0.9375,
+      "loss": 325670.125,
+      "reward": 5.047532066702843,
+      "reward_std": 1.65645283437334,
+      "rewards/concensus_correctness_reward_func": 1.4913125112652779,
+      "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.7815755777992308,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.9084382466971874,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0891249626874924,
+      "rewards/xmlcount_reward_func": 1.14778121560812,
       "step": 6
     },
     {
-      "completion_length": 201.66666666666666,
+      "completion_length": 198.25,
       "epoch": 2.0,
-      "grad_norm": 919.7850952148438,
-      "kl": 161.2150519406423,
+      "grad_norm": 127.43610382080078,
+      "kl": 249.99364541253695,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.1209,
-      "reward": 4.06045925617218,
-      "reward_std": 1.2716497050908704,
-      "rewards/concensus_correctness_reward_func": 0.946083332101504,
-      "rewards/consensus_reward_func": 1.0833333333333333,
+      "loss": 0.1875,
+      "reward": 4.184616843859355,
+      "reward_std": 1.6284607624014218,
+      "rewards/concensus_correctness_reward_func": 0.9245833307504654,
+      "rewards/consensus_reward_func": 0.9166666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8645008318126202,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.8801168352365494,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1665416359901428,
+      "rewards/xmlcount_reward_func": 1.1299166480700176,
       "step": 8
     },
     {
-      "completion_length": 183.71875,
+      "completion_length": 198.25,
       "epoch": 2.571428571428571,
-      "grad_norm": 40.05714416503906,
-      "kl": 0.8664548732340336,
+      "grad_norm": 688616.0625,
+      "kl": 13710.585452570347,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0009,
-      "reward": 4.394081279635429,
-      "reward_std": 1.4197995967697352,
-      "rewards/concensus_correctness_reward_func": 1.2775000035762787,
-      "rewards/consensus_reward_func": 0.9375,
+      "loss": 13.7106,
+      "reward": 4.3517263531684875,
+      "reward_std": 0.8216343149397289,
+      "rewards/concensus_correctness_reward_func": 1.052062500268221,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8390187863260508,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.7998200636357069,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.152562454342842,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 1.1092187277972698,
       "step": 10
     },
     {
-      "completion_length": 150.08333333333334,
+      "completion_length": 162.125,
       "epoch": 3.0,
-      "grad_norm": 92.63465118408203,
-      "kl": 25.041523899572592,
+      "grad_norm": 149.4679412841797,
+      "kl": 1612.202640593052,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0188,
-      "reward": 4.733736574649811,
-      "reward_std": 0.5318293927509027,
-      "rewards/concensus_correctness_reward_func": 1.179583340883255,
+      "loss": 1.2092,
+      "reward": 4.700447857379913,
+      "reward_std": 1.4627919252961874,
+      "rewards/concensus_correctness_reward_func": 1.1300833473602931,
       "rewards/consensus_reward_func": 1.1666666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9672365337610245,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.905281183620294,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1702499836683273,
+      "rewards/xmlcount_reward_func": 1.1650833090146382,
       "step": 12
     },
     {
-      "completion_length": 200.78125,
+      "completion_length": 170.03125,
       "epoch": 3.571428571428571,
-      "grad_norm": 53.50026321411133,
-      "kl": 5.9081552950665355,
+      "grad_norm": 11263439.0,
+      "kl": 224754.29612435307,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0059,
-      "reward": 4.871615186333656,
-      "reward_std": 1.0250823045498691,
-      "rewards/concensus_correctness_reward_func": 1.3903750032186508,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 224.7543,
+      "reward": 4.738578036427498,
+      "reward_std": 1.2284020533552393,
+      "rewards/concensus_correctness_reward_func": 1.2258750051259995,
+      "rewards/consensus_reward_func": 1.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.7810213509947062,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.79782800283283,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 1.168968740850687,
+      "rewards/xmlcount_reward_func": 1.121124967932701,
       "step": 14
     },
     {
-      "completion_length": 184.91666666666666,
+      "completion_length": 178.58333333333334,
       "epoch": 4.0,
-      "grad_norm": 14.596282958984375,
-      "kl": 798915.8579720805,
+      "grad_norm": 43253.8359375,
+      "kl": 173042.15799180977,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 599.187,
-      "reward": 3.898353854815165,
-      "reward_std": 1.4553599038626999,
-      "rewards/concensus_correctness_reward_func": 1.0139999985694885,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "loss": 129.7817,
+      "reward": 4.511672536532084,
+      "reward_std": 1.2354269813901435,
+      "rewards/concensus_correctness_reward_func": 1.2549166679382324,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.9064788594841957,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8479224542776743,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.144541621208191,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 1.1379999617735546,
       "step": 16
     },
     {
-      "completion_length": 182.375,
+      "completion_length": 182.21875,
       "epoch": 4.571428571428571,
-      "grad_norm": 29.889917373657227,
-      "kl": 103274233462938.56,
+      "grad_norm": 21.477466583251953,
+      "kl": 0.7360201131086797,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 103274242048.0,
-      "reward": 4.935375541448593,
-      "reward_std": 0.8632625974641996,
-      "rewards/concensus_correctness_reward_func": 1.3756250031292439,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0007,
+      "reward": 4.0686356127262115,
+      "reward_std": 0.8716689400025643,
+      "rewards/concensus_correctness_reward_func": 0.9155000112950802,
+      "rewards/consensus_reward_func": 0.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8951254896819592,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8094793166965246,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1646249555051327,
+      "rewards/xmlcount_reward_func": 1.1561562269926071,
       "step": 18
     },
     {
-      "completion_length": 189.0,
+      "completion_length": 185.95833333333334,
       "epoch": 5.0,
-      "grad_norm": 20.365131378173828,
-      "kl": 1.5330790672451258,
+      "grad_norm": 207223.09375,
+      "kl": 3623.5394319261736,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0011,
-      "reward": 4.694079021612803,
-      "reward_std": 0.9706238526850939,
-      "rewards/concensus_correctness_reward_func": 1.5170000046491623,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 2.7177,
+      "reward": 4.139020681381226,
+      "reward_std": 1.1705253622494638,
+      "rewards/concensus_correctness_reward_func": 1.1034166763226192,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.8219957128167152,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.8349372744560242,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1884166598320007,
+      "rewards/xmlcount_reward_func": 1.1173333028952281,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 10327424264.741564,
-      "train_runtime": 160.9363,
-      "train_samples_per_second": 1.988,
-      "train_steps_per_second": 0.124
+      "train_loss": 32604.248746631725,
+      "train_runtime": 165.2741,
+      "train_samples_per_second": 1.936,
+      "train_steps_per_second": 0.121
     }
   ],
   "logging_steps": 2,