End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1346.7651600663435,
-    "train_runtime": 114.7259,
+    "train_loss": 0.12956260753126117,
+    "train_runtime": 123.8534,
     "train_samples": 28,
-    "train_samples_per_second": 2.789,
-    "train_steps_per_second": 0.174
+    "train_samples_per_second": 2.584,
+    "train_steps_per_second": 0.161
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd4f0d99b73e85ae32a03c5b29a6f6f7cbdc27edfc00f7c42f043dd407c5ebb2
+oid sha256:19b9b6f34a86f4c7236d477f7ba60b1c001b120b58c0abea0fa85431e70e8c5c
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1346.7651600663435,
-    "train_runtime": 114.7259,
+    "train_loss": 0.12956260753126117,
+    "train_runtime": 123.8534,
     "train_samples": 28,
-    "train_samples_per_second": 2.789,
-    "train_steps_per_second": 0.174
+    "train_samples_per_second": 2.584,
+    "train_steps_per_second": 0.161
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 254.6875,
+      "completion_length": 249.875,
       "epoch": 0.5714285714285714,
-      "grad_norm": 49.11457824707031,
+      "grad_norm": 47.85331726074219,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 3.2836188105866313,
-      "reward_std": 0.7263518713589292,
-      "rewards/concensus_correctness_reward_func": 0.7724999897181988,
-      "rewards/consensus_reward_func": 1.0625,
+      "loss": 0.0,
+      "reward": 3.0323707722127438,
+      "reward_std": 0.8131099180318415,
+      "rewards/concensus_correctness_reward_func": 0.6197499856352806,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4882438706699759,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 0.7884999979287386,
+      "rewards/question_recreation_reward_func": 0.6093395496718585,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.15625,
+      "rewards/xmlcount_reward_func": 0.7564062550663948,
       "step": 2
     },
     {
-      "completion_length": 164.83333333333334,
+      "completion_length": 149.91666666666666,
       "epoch": 1.0,
-      "grad_norm": 79.6348876953125,
-      "kl": 1.8822873709723353,
+      "grad_norm": 103.94192504882812,
+      "kl": 0.13347215698255846,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0014,
-      "reward": 5.652250826358795,
-      "reward_std": 0.6764374932196612,
-      "rewards/concensus_correctness_reward_func": 1.8013333280881245,
+      "loss": 0.0001,
+      "reward": 5.739850004514058,
+      "reward_std": 0.6970982489486536,
+      "rewards/concensus_correctness_reward_func": 1.7566666553417842,
       "rewards/consensus_reward_func": 1.5833333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.7254591931899389,
+      "rewards/question_recreation_reward_func": 0.8079750432322422,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2916666666666667,
-      "rewards/xmlcount_reward_func": 1.1671250015497208,
+      "rewards/strict_format_reward_func": 0.3333333333333333,
+      "rewards/xmlcount_reward_func": 1.175208330154419,
       "step": 4
     },
     {
-      "completion_length": 169.1875,
+      "completion_length": 158.9375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 916771200.0,
-      "kl": 13428740.037678678,
+      "grad_norm": 89.15827941894531,
+      "kl": 1.9806956076063216,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 13428.7402,
-      "reward": 5.615537449717522,
-      "reward_std": 0.5303303725086153,
-      "rewards/concensus_correctness_reward_func": 1.7261249907314777,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.002,
+      "reward": 5.2741532772779465,
+      "reward_std": 0.8553773319144966,
+      "rewards/concensus_correctness_reward_func": 1.6266249865293503,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.825818732380867,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.7314033512957394,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.141718752682209,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 1.1192499995231628,
       "step": 6
     },
     {
-      "completion_length": 166.75,
+      "completion_length": 211.54166666666666,
       "epoch": 2.0,
-      "grad_norm": 42.46098709106445,
-      "kl": 4.602054620782535,
+      "grad_norm": 115.98817443847656,
+      "kl": 8.089183079699675,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0035,
-      "reward": 5.048217097918193,
-      "reward_std": 0.8972721509635448,
-      "rewards/concensus_correctness_reward_func": 1.2505833208560944,
-      "rewards/consensus_reward_func": 1.6666666666666667,
+      "loss": 0.0061,
+      "reward": 4.624983638525009,
+      "reward_std": 0.8529471913352609,
+      "rewards/concensus_correctness_reward_func": 1.2500833123922348,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6336337703590592,
+      "rewards/question_recreation_reward_func": 0.6848169888059298,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3541666666666667,
-      "rewards/xmlcount_reward_func": 1.1431666612625122,
+      "rewards/strict_format_reward_func": 0.14583333333333334,
+      "rewards/xmlcount_reward_func": 1.0442500114440918,
       "step": 8
     },
     {
-      "completion_length": 161.59375,
+      "completion_length": 158.53125,
       "epoch": 2.571428571428571,
-      "grad_norm": 37.895320892333984,
-      "kl": 38800.4430101281,
+      "grad_norm": 128.32345581054688,
+      "kl": 10.754117728210986,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 38.8004,
-      "reward": 5.324525937438011,
-      "reward_std": 0.8662179499806371,
-      "rewards/concensus_correctness_reward_func": 1.4884374886751175,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0108,
+      "reward": 4.959709584712982,
+      "reward_std": 1.3167340854997747,
+      "rewards/concensus_correctness_reward_func": 1.4026249796152115,
+      "rewards/consensus_reward_func": 1.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.8118072021752596,
+      "rewards/question_recreation_reward_func": 0.7500846465118229,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.1649062596261501,
+      "rewards/xmlcount_reward_func": 1.135124996304512,
       "step": 10
     },
     {
-      "completion_length": 155.54166666666666,
+      "completion_length": 184.33333333333334,
       "epoch": 3.0,
-      "grad_norm": 92.2210922241211,
-      "kl": 11.152309625719985,
+      "grad_norm": 24.66672706604004,
+      "kl": 3.1319144380589328,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0084,
-      "reward": 5.643816947937012,
-      "reward_std": 0.5692423766789337,
-      "rewards/concensus_correctness_reward_func": 1.6060833235581715,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0023,
+      "reward": 5.048190355300903,
+      "reward_std": 0.7422210735579332,
+      "rewards/concensus_correctness_reward_func": 1.428333322207133,
+      "rewards/consensus_reward_func": 1.6666666666666667,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8004835819204649,
+      "rewards/question_recreation_reward_func": 0.7871903777122498,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3541666666666667,
-      "rewards/xmlcount_reward_func": 1.1330833385388057,
+      "rewards/strict_format_reward_func": 0.20833333333333334,
+      "rewards/xmlcount_reward_func": 0.9576666702826818,
       "step": 12
     },
     {
-      "completion_length": 163.46875,
+      "completion_length": 210.53125,
       "epoch": 3.571428571428571,
-      "grad_norm": 167.47503662109375,
-      "kl": 13.075925993267447,
+      "grad_norm": 440.6236267089844,
+      "kl": 37.00089144241065,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0131,
-      "reward": 5.369014501571655,
-      "reward_std": 0.4032270271272864,
-      "rewards/concensus_correctness_reward_func": 1.3764999955892563,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.037,
+      "reward": 4.599987611174583,
+      "reward_std": 1.3417337444698205,
+      "rewards/concensus_correctness_reward_func": 1.2383124865591526,
+      "rewards/consensus_reward_func": 1.4375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.896764550358057,
+      "rewards/question_recreation_reward_func": 0.7561751045286655,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.2207500040531158,
+      "rewards/strict_format_reward_func": 0.203125,
+      "rewards/xmlcount_reward_func": 0.9648750014603138,
       "step": 14
     },
     {
-      "completion_length": 167.125,
+      "completion_length": 165.45833333333334,
       "epoch": 4.0,
-      "grad_norm": 154.0272674560547,
-      "kl": 99.95235991602142,
+      "grad_norm": 884.29345703125,
+      "kl": 153.8786713940402,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.075,
-      "reward": 4.875289618968964,
-      "reward_std": 1.0041625554828595,
-      "rewards/concensus_correctness_reward_func": 1.3239999810854595,
-      "rewards/consensus_reward_func": 1.4166666666666667,
+      "loss": 0.1154,
+      "reward": 5.407021721204122,
+      "reward_std": 0.9267256280872971,
+      "rewards/concensus_correctness_reward_func": 1.5748333086570103,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.7283729445189238,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.7176050413399935,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2708333333333333,
-      "rewards/xmlcount_reward_func": 1.0520833333333333,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.1354166666666667,
       "step": 16
     },
     {
-      "completion_length": 157.3125,
+      "completion_length": 167.03125,
       "epoch": 4.571428571428571,
-      "grad_norm": 43.15923309326172,
-      "kl": 3.150787123478949,
+      "grad_norm": 433.138916015625,
+      "kl": 51.80729316617362,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0032,
-      "reward": 5.6706234365701675,
-      "reward_std": 0.9191514646518044,
-      "rewards/concensus_correctness_reward_func": 1.5298749953508377,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0518,
+      "reward": 4.95432373136282,
+      "reward_std": 0.6955515777153778,
+      "rewards/concensus_correctness_reward_func": 1.3982499837875366,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.7447797094937414,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.7722300551831722,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.1928437426686287,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 1.0338437519967556,
       "step": 18
     },
     {
-      "completion_length": 176.58333333333334,
+      "completion_length": 213.45833333333334,
       "epoch": 5.0,
-      "grad_norm": 312.3111877441406,
-      "kl": 8.666269062707821,
+      "grad_norm": 30.303447723388672,
+      "kl": 1426.8845755159855,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0065,
-      "reward": 4.999991357326508,
-      "reward_std": 0.8645469406619668,
-      "rewards/concensus_correctness_reward_func": 1.4022499819596608,
-      "rewards/consensus_reward_func": 1.3333333333333333,
+      "loss": 1.0702,
+      "reward": 4.796473105748494,
+      "reward_std": 0.7849306451777617,
+      "rewards/concensus_correctness_reward_func": 1.431666652361552,
+      "rewards/consensus_reward_func": 1.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7614080148438612,
+      "rewards/question_recreation_reward_func": 0.7149731454749902,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3541666666666667,
-      "rewards/xmlcount_reward_func": 1.148833344380061,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 0.9831666549046835,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1346.7651600663435,
-      "train_runtime": 114.7259,
-      "train_samples_per_second": 2.789,
-      "train_steps_per_second": 0.174
+      "train_loss": 0.12956260753126117,
+      "train_runtime": 123.8534,
+      "train_samples_per_second": 2.584,
+      "train_steps_per_second": 0.161
     }
   ],
   "logging_steps": 2,