End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.3126821821319027,
-    "train_runtime": 186.5224,
-    "train_samples": 160,
-    "train_samples_per_second": 1.716,
-    "train_steps_per_second": 0.107
+    "train_loss": 5.050748984558595e-06,
+    "train_runtime": 778.5481,
+    "train_samples": 320,
+    "train_samples_per_second": 0.411,
+    "train_steps_per_second": 0.026
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8a4a37ab54474338c92a54016fdd18df71d926a436e4be9a21931fe055a92b5
+oid sha256:717ed586af818b332fca9a378af8befd7f9936da66468acc9efe1cabc9612e89
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.3126821821319027,
-    "train_runtime": 186.5224,
-    "train_samples": 160,
-    "train_samples_per_second": 1.716,
-    "train_steps_per_second": 0.107
+    "train_loss": 5.050748984558595e-06,
+    "train_runtime": 778.5481,
+    "train_samples": 320,
+    "train_samples_per_second": 0.411,
+    "train_steps_per_second": 0.026
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.5,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 258.125,
-      "epoch": 0.1,
-      "grad_norm": 12.1038236618042,
+      "completion_length": 282.71875,
+      "epoch": 0.05,
+      "grad_norm": 42.93820571899414,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 2.3500917481724173,
-      "reward_std": 0.9744344543432817,
-      "rewards/concensus_correctness_reward_func": 0.5234999991953373,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 0.6292939200648107,
+      "reward_std": 0.6506021634559147,
+      "rewards/concensus_correctness_reward_func": 0.04662499949336052,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5244667693041265,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.14407516870414838,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 0.5677500029560179,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.1260937498882413,
       "step": 2
     },
     {
-      "completion_length": 157.28125,
-      "epoch": 0.2,
-      "grad_norm": 15.164268493652344,
-      "kl": 0.016867930413582144,
+      "completion_length": 316.15625,
+      "epoch": 0.1,
+      "grad_norm": 11.557947158813477,
+      "kl": 0.002068059044177062,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 4.654461152851582,
-      "reward_std": 1.1400904605397955,
-      "rewards/concensus_correctness_reward_func": 1.1535625024698675,
-      "rewards/consensus_reward_func": 1.375,
+      "reward": 0.5142579441308044,
+      "reward_std": 0.40489116005483083,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6927111456170678,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.22807045001536608,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.1363125015050173,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.09868749883025885,
       "step": 4
     },
     {
-      "completion_length": 170.4375,
-      "epoch": 0.3,
-      "grad_norm": 11.745673179626465,
-      "kl": 0.31349003181094304,
+      "completion_length": 397.90625,
+      "epoch": 0.15,
+      "grad_norm": 33.66463851928711,
+      "kl": 0.0023345137960859574,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0003,
-      "reward": 5.0158578380942345,
-      "reward_std": 2.04010990288225,
-      "rewards/concensus_correctness_reward_func": 1.4669375028461218,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 0.0876373503706418,
+      "reward_std": 0.2730903436749941,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.7887953128665686,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.12148110551061109,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.1507500000298023,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.0338437519967556,
       "step": 6
     },
     {
-      "completion_length": 140.59375,
-      "epoch": 0.4,
-      "grad_norm": 450.44647216796875,
-      "kl": 18.68448106572032,
+      "completion_length": 322.21875,
+      "epoch": 0.2,
+      "grad_norm": 13.174933433532715,
+      "kl": 0.0018492022645659745,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0187,
-      "reward": 4.864694945514202,
-      "reward_std": 1.9919630960794166,
-      "rewards/concensus_correctness_reward_func": 1.2613750007003546,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 0.41745682479813695,
+      "reward_std": 0.3582948070834391,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.6198824238963425,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.1773943287844304,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1709375083446503,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.052562499884516,
       "step": 8
     },
     {
-      "completion_length": 131.8125,
-      "epoch": 0.5,
-      "grad_norm": 47933.35546875,
-      "kl": 3090.915692106355,
+      "completion_length": 288.125,
+      "epoch": 0.25,
+      "grad_norm": 67.86888122558594,
+      "kl": 0.008549842281354358,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 3.0909,
-      "reward": 4.455410748720169,
-      "reward_std": 0.9344652489380678,
-      "rewards/concensus_correctness_reward_func": 1.055249996483326,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 0.40863744184025563,
+      "reward_std": 0.49519592405704316,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7835044432431459,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.1976999432372395,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.1322812512516975,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.07031250046566129,
       "step": 10
     },
     {
-      "completion_length": 170.25,
-      "epoch": 0.6,
-      "grad_norm": 50.33361053466797,
-      "kl": 5.447311662370339,
+      "completion_length": 212.03125,
+      "epoch": 0.3,
+      "grad_norm": 24.831544876098633,
+      "kl": 0.008816897239739774,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0054,
-      "reward": 4.70535284280777,
-      "reward_std": 1.5298584499396384,
-      "rewards/concensus_correctness_reward_func": 1.2123750075697899,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0,
+      "reward": 0.36399466946022585,
+      "reward_std": 0.2636154612409882,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6975716073065996,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.18833842262392864,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.1860312521457672,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.17565625021234155,
       "step": 12
     },
     {
-      "completion_length": 132.25,
-      "epoch": 0.7,
-      "grad_norm": 59.70350646972656,
-      "kl": 6.450560495257378,
+      "completion_length": 369.34375,
+      "epoch": 0.35,
+      "grad_norm": 4.622942924499512,
+      "kl": 0.002492460354915238,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0065,
-      "reward": 4.6385799795389175,
-      "reward_std": 1.0262601334834471,
-      "rewards/concensus_correctness_reward_func": 1.1698749922215939,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0,
+      "reward": 0.13834899943321943,
+      "reward_std": 0.13620900230307598,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6355173867195845,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.07897400224464945,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.145687498152256,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.059374999604187906,
       "step": 14
     },
     {
-      "completion_length": 142.03125,
-      "epoch": 0.8,
-      "grad_norm": 21.690887451171875,
-      "kl": 2.6143888300284743,
+      "completion_length": 334.78125,
+      "epoch": 0.4,
+      "grad_norm": 42.03730010986328,
+      "kl": 0.005604283696811763,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0026,
-      "reward": 3.90111280977726,
-      "reward_std": 1.5290619616862386,
-      "rewards/concensus_correctness_reward_func": 0.7911250032484531,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0,
+      "reward": 1.140276842750609,
+      "reward_std": 0.9167491418556892,
+      "rewards/concensus_correctness_reward_func": 0.7786874994635582,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6498003372689709,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.11102682136697695,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.1633124947547913,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.06306249997578561,
       "step": 16
     },
     {
-      "completion_length": 130.65625,
-      "epoch": 0.9,
-      "grad_norm": 14.880861282348633,
-      "kl": 0.7605363634356763,
+      "completion_length": 370.53125,
+      "epoch": 0.45,
+      "grad_norm": 103.69042205810547,
+      "kl": 0.016677564308338333,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0008,
-      "reward": 5.188799202442169,
-      "reward_std": 1.0886195020284504,
-      "rewards/concensus_correctness_reward_func": 1.276500005275011,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0,
+      "reward": 0.251421230728738,
+      "reward_std": 0.507526803878136,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7756116222590208,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.2050462217302993,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.214812494814396,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.01612500031478703,
       "step": 18
     },
     {
-      "completion_length": 131.8125,
-      "epoch": 1.0,
-      "grad_norm": 21.948577880859375,
-      "kl": 1.6171908722026274,
+      "completion_length": 309.03125,
+      "epoch": 0.5,
+      "grad_norm": 18.9988956451416,
+      "kl": 0.0024696916407265235,
       "learning_rate": 0.0,
-      "loss": 0.0016,
-      "reward": 5.907395571470261,
-      "reward_std": 1.0099625813309103,
-      "rewards/concensus_correctness_reward_func": 1.6532500013709068,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0,
+      "reward": 0.4606377884047106,
+      "reward_std": 0.46051671362511115,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.7589581478387117,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.16510653705336154,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.2139374986290932,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.1549062510021031,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.5,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.3126821821319027,
-      "train_runtime": 186.5224,
-      "train_samples_per_second": 1.716,
-      "train_steps_per_second": 0.107
+      "train_loss": 5.050748984558595e-06,
+      "train_runtime": 778.5481,
+      "train_samples_per_second": 0.411,
+      "train_steps_per_second": 0.026
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2733e682c379da13a7f0f60bf6a3ddae800b8081537511973c8d1257a112736a
+oid sha256:06ef677395a87ab90b386275bafabbb7b44c577ab992a61abdc3c6cb010f9a75
 size 6008