End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.693535439635161,
-    "train_runtime": 167.0439,
-    "train_samples": 160,
-    "train_samples_per_second": 1.916,
-    "train_steps_per_second": 0.12
+    "train_loss": 2.675073219506885e-05,
+    "train_runtime": 372.7551,
+    "train_samples": 240,
+    "train_samples_per_second": 0.858,
+    "train_steps_per_second": 0.054
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a48784e434429cc978157f1307d0a88d5da208f024eff11f3339fd7b0f189dff
+oid sha256:7c257783271822b61103458e5685bed66906947058c2d7e928a08e149d4fbb89
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.693535439635161,
-    "train_runtime": 167.0439,
-    "train_samples": 160,
-    "train_samples_per_second": 1.916,
-    "train_steps_per_second": 0.12
+    "train_loss": 2.675073219506885e-05,
+    "train_runtime": 372.7551,
+    "train_samples": 240,
+    "train_samples_per_second": 0.858,
+    "train_steps_per_second": 0.054
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.6666666666666666,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 227.53125,
-      "epoch": 0.1,
-      "grad_norm": 30.49525260925293,
+      "completion_length": 348.53125,
+      "epoch": 0.06666666666666667,
+      "grad_norm": 59.67878341674805,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": -0.0,
-      "reward": 4.441966446116567,
-      "reward_std": 1.1807270474964753,
-      "rewards/concensus_correctness_reward_func": 1.641249991953373,
-      "rewards/consensus_reward_func": 1.0,
+      "reward": 1.572420941607561,
+      "reward_std": 2.047676354914074,
+      "rewards/concensus_correctness_reward_func": 1.25,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6907477553468198,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.1417647238704376,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 0.8755937553360127,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.055656248703598976,
       "step": 2
     },
     {
-      "completion_length": 144.65625,
-      "epoch": 0.2,
-      "grad_norm": 41.56974792480469,
-      "kl": 2.6795606416417286,
+      "completion_length": 324.65625,
+      "epoch": 0.13333333333333333,
+      "grad_norm": 131.75711059570312,
+      "kl": 0.018102863609783526,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0027,
-      "reward": 6.809520989656448,
-      "reward_std": 0.14137882058275864,
-      "rewards/concensus_correctness_reward_func": 1.9741249680519104,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.0,
+      "reward": 0.31940190950990655,
+      "reward_std": 0.44206382022821344,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9491771869361401,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.1256206654070411,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.229968748986721,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.05621875077486038,
       "step": 4
     },
     {
-      "completion_length": 164.90625,
-      "epoch": 0.3,
-      "grad_norm": 58.901611328125,
-      "kl": 3212.812270720955,
+      "completion_length": 247.03125,
+      "epoch": 0.2,
+      "grad_norm": 71.89510345458984,
+      "kl": 0.0316093881483539,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 3.2128,
-      "reward": 6.152694694697857,
-      "reward_std": 0.2201200993731618,
-      "rewards/concensus_correctness_reward_func": 1.727374978363514,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0,
+      "reward": 1.1185205472283997,
+      "reward_std": 1.4224079721607268,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9292259570211172,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.1053643089835532,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.16796875,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.07565625105053186,
       "step": 6
     },
     {
-      "completion_length": 156.0625,
-      "epoch": 0.4,
-      "grad_norm": 21159.541015625,
-      "kl": 793.1982030207291,
+      "completion_length": 290.875,
+      "epoch": 0.26666666666666666,
+      "grad_norm": 9.459587097167969,
+      "kl": 0.02046902662186767,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.7932,
-      "reward": 6.348659709095955,
-      "reward_std": 0.6525374334305525,
-      "rewards/concensus_correctness_reward_func": 1.9973749844357371,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 0.0,
+      "reward": 0.2236626870580949,
+      "reward_std": 0.5096143184637185,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8731908686459064,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.13541268536937423,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.1187187507748604,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.03674999997019768,
       "step": 8
     },
     {
-      "completion_length": 166.71875,
-      "epoch": 0.5,
-      "grad_norm": 3487.975830078125,
-      "kl": 235.72644520667382,
+      "completion_length": 334.6875,
+      "epoch": 0.3333333333333333,
+      "grad_norm": 84.37945556640625,
+      "kl": 0.04284666738749365,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.2357,
-      "reward": 6.8310661762952805,
-      "reward_std": 0.966264896094799,
-      "rewards/concensus_correctness_reward_func": 2.3566250018775463,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 0.0,
+      "reward": 1.155805722461082,
+      "reward_std": 1.1361800509039313,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.8903785869479179,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.1522432168130763,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.0059375017881393,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.06606250046752393,
       "step": 10
     },
     {
-      "completion_length": 149.9375,
-      "epoch": 0.6,
-      "grad_norm": 1146.8641357421875,
-      "kl": 125.67277173465118,
+      "completion_length": 212.09375,
+      "epoch": 0.4,
+      "grad_norm": 30.05791664123535,
+      "kl": 0.003935115470085293,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.1257,
-      "reward": 6.07975634932518,
-      "reward_std": 0.34126558783464134,
-      "rewards/concensus_correctness_reward_func": 1.8357499688863754,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0,
+      "reward": 0.1393712537828833,
+      "reward_std": 0.3741129007685231,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8660688688978553,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.10759001411497593,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1904375031590462,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.09321875125169754,
       "step": 12
     },
     {
-      "completion_length": 141.8125,
-      "epoch": 0.7,
-      "grad_norm": 28.320093154907227,
-      "kl": 42091.95313343528,
+      "completion_length": 284.4375,
+      "epoch": 0.4666666666666667,
+      "grad_norm": 36.2635383605957,
+      "kl": 0.010665363864973187,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 42.092,
-      "reward": 6.489867955446243,
-      "reward_std": 0.31974472710862756,
-      "rewards/concensus_correctness_reward_func": 2.0317499861121178,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0,
+      "reward": 1.1181357721798122,
+      "reward_std": 1.3259522412138267,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.9453992135822773,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.06760448752902448,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.2002187445759773,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.05053124995902181,
       "step": 14
     },
     {
-      "completion_length": 150.53125,
-      "epoch": 0.8,
-      "grad_norm": 46.18184280395508,
-      "kl": 1.4570958691183478,
+      "completion_length": 272.5625,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 63.69668960571289,
+      "kl": 0.030998096106486628,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0015,
-      "reward": 6.744259178638458,
-      "reward_std": 0.062466773204505444,
-      "rewards/concensus_correctness_reward_func": 1.9677499830722809,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.0,
+      "reward": 0.16769864893285558,
+      "reward_std": 0.1935645243502222,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9749154075980186,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.12938614611630328,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.2390937432646751,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.024187501519918442,
       "step": 16
     },
     {
-      "completion_length": 143.65625,
-      "epoch": 0.9,
-      "grad_norm": 19582.970703125,
-      "kl": 464.659947514534,
+      "completion_length": 357.40625,
+      "epoch": 0.6,
+      "grad_norm": 147.51528930664062,
+      "kl": 0.048071805846120697,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.4647,
-      "reward": 6.67893061041832,
-      "reward_std": 0.07036678654549178,
-      "rewards/concensus_correctness_reward_func": 2.0833749808371067,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0,
+      "reward": 0.21695712665678002,
+      "reward_std": 0.1947699283482507,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.9845244064927101,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.16617587706423365,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.189156249165535,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.05078125046566129,
       "step": 18
     },
     {
-      "completion_length": 143.5625,
-      "epoch": 1.0,
-      "grad_norm": 28.336759567260742,
-      "kl": 7.188884501068969,
+      "completion_length": 282.34375,
+      "epoch": 0.6666666666666666,
+      "grad_norm": 199.55126953125,
+      "kl": 0.06104159675669507,
       "learning_rate": 0.0,
-      "loss": 0.0072,
-      "reward": 6.240718752145767,
-      "reward_std": 0.3274476360529661,
-      "rewards/concensus_correctness_reward_func": 1.8999999836087227,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0001,
+      "reward": 1.0380627679405734,
+      "reward_std": 1.3433483467670158,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8931874856352806,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.12543774559162557,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.181906245648861,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.037625000346452,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.6666666666666666,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 4.693535439635161,
-      "train_runtime": 167.0439,
-      "train_samples_per_second": 1.916,
-      "train_steps_per_second": 0.12
+      "train_loss": 2.675073219506885e-05,
+      "train_runtime": 372.7551,
+      "train_samples_per_second": 0.858,
+      "train_steps_per_second": 0.054
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93ce4e3fb722178fdf50853e64d75b7d233c7f29bd5c6c990abe0fda15b02a2e
+oid sha256:dd2693631c8676f8590d2229a65da92d473e61fb9110e074977e993485ea2788
 size 6008