End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae9b964b52f1623aafe3252fa477f7a70e68b2f77a85d0b845c8d5ada7700982
+oid sha256:c71c43c33e73e15fdadb7b1d584af9d29319a3b7c0ca232d3324fe5ccb77eb1b
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0030404651071876286,
-    "train_runtime": 261.6033,
+    "train_loss": 0.014572195638902486,
+    "train_runtime": 240.5248,
     "train_samples": 14,
-    "train_samples_per_second": 1.223,
-    "train_steps_per_second": 0.076
+    "train_samples_per_second": 1.33,
+    "train_steps_per_second": 0.083
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0030404651071876286,
-    "train_runtime": 261.6033,
+    "train_loss": 0.014572195638902486,
+    "train_runtime": 240.5248,
     "train_samples": 14,
-    "train_samples_per_second": 1.223,
-    "train_steps_per_second": 0.076
+    "train_samples_per_second": 1.33,
+    "train_steps_per_second": 0.083
 }

trainer_state.json

@@ -9,203 +9,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 252.6875,
+      "completion_length": 239.9375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 2.701991081237793,
-      "kl": 4.37556079775095,
+      "grad_norm": 22.087520599365234,
+      "kl": 23.55493902415037,
       "learning_rate": 4.965903258506806e-07,
-      "loss": 0.0044,
-      "reward": 12.043715000152588,
-      "reward_std": 7.9242489486932755,
-      "rewards/concensus_correctness_reward_func": 8.318812549114227,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0236,
+      "reward": 6.47322241961956,
+      "reward_std": 6.425537696108222,
+      "rewards/concensus_correctness_reward_func": 3.2521250061690807,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.6523090489208698,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.5753786675632,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 0.8382187411189079,
+      "rewards/strict_format_reward_func": 0.21875,
+      "rewards/xmlcount_reward_func": 0.9269687570631504,
       "step": 2
     },
     {
-      "completion_length": 232.33333333333334,
+      "completion_length": 229.0,
       "epoch": 1.0,
-      "grad_norm": 1.2891075611114502,
-      "kl": 4.701668918132782,
+      "grad_norm": 1.0740150213241577,
+      "kl": 4.5665431916713715,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0035,
-      "reward": 12.15744169553121,
-      "reward_std": 11.759601910909018,
-      "rewards/concensus_correctness_reward_func": 7.79283332824707,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.0034,
+      "reward": 15.436209201812744,
+      "reward_std": 9.165850599606832,
+      "rewards/concensus_correctness_reward_func": 10.720249811808268,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.7228167454401652,
+      "rewards/final_correctness_reward_func": 1.6666666666666667,
+      "rewards/question_recreation_reward_func": 0.6404593139886856,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3333333333333333,
-      "rewards/xmlcount_reward_func": 1.141791671514511,
+      "rewards/strict_format_reward_func": 0.3541666666666667,
+      "rewards/xmlcount_reward_func": 1.0546666582425435,
       "step": 4
     },
     {
-      "completion_length": 253.375,
+      "completion_length": 233.90625,
       "epoch": 1.5714285714285714,
-      "grad_norm": 2.646155595779419,
-      "kl": 1.9060122929513454,
+      "grad_norm": 9.612006187438965,
+      "kl": 14.76283297687769,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0019,
-      "reward": 13.61039674282074,
-      "reward_std": 8.232968762516975,
-      "rewards/concensus_correctness_reward_func": 8.952187538146973,
-      "rewards/consensus_reward_func": 1.0625,
+      "loss": 0.0148,
+      "reward": 11.3202466070652,
+      "reward_std": 4.755059517920017,
+      "rewards/concensus_correctness_reward_func": 6.97112499922514,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4375,
-      "rewards/question_recreation_reward_func": 0.7582718506455421,
+      "rewards/final_correctness_reward_func": 1.3125,
+      "rewards/question_recreation_reward_func": 0.5594028290361166,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.0561875253915787,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.0397187694907188,
       "step": 6
     },
     {
-      "completion_length": 242.75,
+      "completion_length": 192.20833333333334,
       "epoch": 2.0,
-      "grad_norm": 2.2974019050598145,
-      "kl": 3.705540662010511,
+      "grad_norm": 2.1542346477508545,
+      "kl": 6.417887091636658,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0028,
-      "reward": 15.746721982955933,
-      "reward_std": 4.242142866055171,
-      "rewards/concensus_correctness_reward_func": 11.126416683197021,
+      "loss": 0.0048,
+      "reward": 14.476815938949585,
+      "reward_std": 6.148909568786621,
+      "rewards/concensus_correctness_reward_func": 10.143666585286459,
       "rewards/consensus_reward_func": 1.1666666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.7123052279154459,
+      "rewards/final_correctness_reward_func": 1.1666666666666667,
+      "rewards/question_recreation_reward_func": 0.5249826436241468,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3333333333333333,
-      "rewards/xmlcount_reward_func": 1.0746666689713795,
+      "rewards/strict_format_reward_func": 0.3541666666666667,
+      "rewards/xmlcount_reward_func": 1.1206666827201843,
       "step": 8
     },
     {
-      "completion_length": 297.5,
+      "completion_length": 240.25,
       "epoch": 2.571428571428571,
-      "grad_norm": 1.8097786903381348,
-      "kl": 3.4749511815607548,
+      "grad_norm": 2.1911938190460205,
+      "kl": 45.11784844473004,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0035,
-      "reward": 13.5066457092762,
-      "reward_std": 6.842143721878529,
-      "rewards/concensus_correctness_reward_func": 9.23956248909235,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0451,
+      "reward": 11.117486327886581,
+      "reward_std": 7.315313369035721,
+      "rewards/concensus_correctness_reward_func": 7.300437584519386,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1875,
-      "rewards/question_recreation_reward_func": 0.8563956692814827,
+      "rewards/final_correctness_reward_func": 1.0625,
+      "rewards/question_recreation_reward_func": 0.6240489408373833,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 0.9263125211000443,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.0054999962449074,
       "step": 10
     },
     {
-      "completion_length": 244.08333333333334,
+      "completion_length": 219.58333333333334,
       "epoch": 3.0,
-      "grad_norm": 1.1800462007522583,
-      "kl": 2.213648021221161,
+      "grad_norm": 15.332870483398438,
+      "kl": 13.44922679165999,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0017,
-      "reward": 10.94917126496633,
-      "reward_std": 9.821666737397512,
-      "rewards/concensus_correctness_reward_func": 6.952083349227905,
-      "rewards/consensus_reward_func": 0.8333333333333334,
+      "loss": 0.0101,
+      "reward": 7.629446824391683,
+      "reward_std": 6.145598113536835,
+      "rewards/concensus_correctness_reward_func": 4.131833344697952,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.6622129753232002,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.5357804199059805,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.2708333333333333,
-      "rewards/xmlcount_reward_func": 0.9807083606719971,
+      "rewards/xmlcount_reward_func": 0.9410000033676624,
       "step": 12
     },
     {
-      "completion_length": 216.84375,
+      "completion_length": 233.59375,
       "epoch": 3.571428571428571,
-      "grad_norm": 2.971205711364746,
-      "kl": 6.725032597780228,
+      "grad_norm": 2.538357734680176,
+      "kl": 6.863416824489832,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0067,
-      "reward": 14.822497367858887,
-      "reward_std": 11.08877694606781,
-      "rewards/concensus_correctness_reward_func": 10.437687516212463,
+      "loss": 0.0069,
+      "reward": 15.271047949790955,
+      "reward_std": 11.11886179447174,
+      "rewards/concensus_correctness_reward_func": 10.532624959945679,
       "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.6994972974061966,
+      "rewards/final_correctness_reward_func": 1.625,
+      "rewards/question_recreation_reward_func": 0.6913608647882938,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.0603125244379044,
+      "rewards/xmlcount_reward_func": 1.0470625050365925,
       "step": 14
     },
     {
-      "completion_length": 231.125,
+      "completion_length": 223.54166666666666,
       "epoch": 4.0,
-      "grad_norm": 0.9656513929367065,
-      "kl": 1.6770356198151906,
+      "grad_norm": 11.851518630981445,
+      "kl": 30.946319967508316,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0013,
-      "reward": 9.871681809425354,
-      "reward_std": 4.748319000005722,
-      "rewards/concensus_correctness_reward_func": 5.983166694641113,
-      "rewards/consensus_reward_func": 1.0833333333333333,
+      "loss": 0.0232,
+      "reward": 7.431079347928365,
+      "reward_std": 5.825171808401744,
+      "rewards/concensus_correctness_reward_func": 4.34958328306675,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.5080151110887527,
+      "rewards/final_correctness_reward_func": 0.5833333333333334,
+      "rewards/question_recreation_reward_func": 0.537370835741361,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2916666666666667,
-      "rewards/xmlcount_reward_func": 1.0054999987284343,
+      "rewards/strict_format_reward_func": 0.2708333333333333,
+      "rewards/xmlcount_reward_func": 0.8566250105698904,
       "step": 16
     },
     {
-      "completion_length": 243.59375,
+      "completion_length": 234.84375,
       "epoch": 4.571428571428571,
-      "grad_norm": 1.8503743410110474,
-      "kl": 3.721465650945902,
+      "grad_norm": 5.189236640930176,
+      "kl": 9.85001516342163,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0037,
-      "reward": 11.102317184209824,
-      "reward_std": 8.393039897084236,
-      "rewards/concensus_correctness_reward_func": 6.8781875520944595,
+      "loss": 0.0099,
+      "reward": 9.794197231531143,
+      "reward_std": 7.727541692554951,
+      "rewards/concensus_correctness_reward_func": 5.730812497437,
       "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.6781611368060112,
+      "rewards/final_correctness_reward_func": 1.375,
+      "rewards/question_recreation_reward_func": 0.6756972223520279,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 0.9834687560796738,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 0.9658125042915344,
       "step": 18
     },
     {
-      "completion_length": 253.125,
+      "completion_length": 247.375,
       "epoch": 5.0,
-      "grad_norm": 1.3199962377548218,
-      "kl": 1.3040494720141094,
+      "grad_norm": 2.1349575519561768,
+      "kl": 5.3841776847839355,
       "learning_rate": 0.0,
-      "loss": 0.001,
-      "reward": 16.325387994448345,
-      "reward_std": 7.758129904667537,
-      "rewards/concensus_correctness_reward_func": 11.791666666666666,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.004,
+      "reward": 11.094345887502035,
+      "reward_std": 9.930945932865143,
+      "rewards/concensus_correctness_reward_func": 7.086333274841309,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.6855962077776591,
+      "rewards/final_correctness_reward_func": 1.1666666666666667,
+      "rewards/question_recreation_reward_func": 0.614970917503039,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 0.9522916873296102,
+      "rewards/strict_format_reward_func": 0.3541666666666667,
+      "rewards/xmlcount_reward_func": 1.038874973853429,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.0030404651071876286,
-      "train_runtime": 261.6033,
-      "train_samples_per_second": 1.223,
-      "train_steps_per_second": 0.076
+      "train_loss": 0.014572195638902486,
+      "train_runtime": 240.5248,
+      "train_samples_per_second": 1.33,
+      "train_steps_per_second": 0.083
     }
   ],
   "logging_steps": 2,