End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.00031505114329775095,
-    "train_runtime": 1048.8673,
-    "train_samples": 197,
-    "train_samples_per_second": 0.305,
-    "train_steps_per_second": 0.019
+    "train_loss": 0.00457829560036771,
+    "train_runtime": 805.5031,
+    "train_samples": 161,
+    "train_samples_per_second": 0.397,
+    "train_steps_per_second": 0.025
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e197570708b7b13c3c45c47e6b280af2936842e7c55027621ebf5169ae627a0
+oid sha256:25f4be18da37ed009b3a825b1861d6c27a947952f19a732fb0d1720782c18587
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.00031505114329775095,
-    "train_runtime": 1048.8673,
-    "train_samples": 197,
-    "train_samples_per_second": 0.305,
-    "train_steps_per_second": 0.019
+    "train_loss": 0.00457829560036771,
+    "train_runtime": 805.5031,
+    "train_samples": 161,
+    "train_samples_per_second": 0.397,
+    "train_steps_per_second": 0.025
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.8121827411167513,
+  "epoch": 0.9937888198757764,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 347.0,
-      "epoch": 0.08121827411167512,
-      "grad_norm": 25.34670639038086,
+      "completion_length": 311.90625,
+      "epoch": 0.09937888198757763,
+      "grad_norm": 16.116872787475586,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
-      "loss": 0.0,
-      "reward": 1.8532088848296553,
-      "reward_std": 0.7197320554842008,
-      "rewards/concensus_correctness_reward_func": 0.42650000005960464,
-      "rewards/consensus_reward_func": 0.4375,
+      "loss": -0.0,
+      "reward": 2.0880651086336,
+      "reward_std": 0.6684938869730104,
+      "rewards/concensus_correctness_reward_func": 0.3713750019669533,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.4665213600965217,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.39662764465902,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.109375,
-      "rewards/xmlcount_reward_func": 0.22581250546500087,
+      "rewards/strict_format_reward_func": 0.15625,
+      "rewards/xmlcount_reward_func": 0.5388124994933605,
       "step": 2
     },
     {
-      "completion_length": 214.8125,
-      "epoch": 0.16243654822335024,
-      "grad_norm": 21.418426513671875,
-      "kl": 0.02307581211789511,
+      "completion_length": 187.03125,
+      "epoch": 0.19875776397515527,
+      "grad_norm": 20.159204483032227,
+      "kl": 0.03585821596789174,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 3.239914685487747,
-      "reward_std": 1.1554774387914222,
-      "rewards/concensus_correctness_reward_func": 0.45393750071525574,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 3.9214092064648867,
+      "reward_std": 1.652882975889952,
+      "rewards/concensus_correctness_reward_func": 0.9944375082850456,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5888834362849593,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5815029521472752,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.1033437550067902,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.173593744635582,
       "step": 4
     },
     {
-      "completion_length": 190.75,
-      "epoch": 0.2436548223350254,
-      "grad_norm": 35.000457763671875,
-      "kl": 0.3683422572212294,
+      "completion_length": 179.375,
+      "epoch": 0.2981366459627329,
+      "grad_norm": 26.64881134033203,
+      "kl": 0.47550190216861665,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0004,
-      "reward": 3.8262907788157463,
-      "reward_std": 1.3872474904637784,
-      "rewards/concensus_correctness_reward_func": 1.1668749935925007,
-      "rewards/consensus_reward_func": 0.5,
+      "loss": 0.0005,
+      "reward": 3.2665678560733795,
+      "reward_std": 1.333875675802119,
+      "rewards/concensus_correctness_reward_func": 0.4598749987781048,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6158845135942101,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.48253656178712845,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1060312539339066,
+      "rewards/strict_format_reward_func": 0.390625,
+      "rewards/xmlcount_reward_func": 1.121031254529953,
       "step": 6
     },
     {
-      "completion_length": 202.8125,
-      "epoch": 0.3248730964467005,
-      "grad_norm": 16.695356369018555,
-      "kl": 0.21090720151551068,
+      "completion_length": 206.5625,
+      "epoch": 0.39751552795031053,
+      "grad_norm": 28.57688331604004,
+      "kl": 0.20366718503646553,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0002,
-      "reward": 3.1408818289637566,
-      "reward_std": 1.3075056065281387,
-      "rewards/concensus_correctness_reward_func": 0.2862499989569187,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 5.219966318458319,
+      "reward_std": 2.007999164707144,
+      "rewards/concensus_correctness_reward_func": 1.7436874955892563,
+      "rewards/consensus_reward_func": 1.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6319755638251081,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.6211538150673732,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.14453125,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.1363750025629997,
       "step": 8
     },
     {
-      "completion_length": 239.71875,
-      "epoch": 0.40609137055837563,
-      "grad_norm": 17.293676376342773,
-      "kl": 1.533459787024185,
+      "completion_length": 169.3125,
+      "epoch": 0.4968944099378882,
+      "grad_norm": 44.238037109375,
+      "kl": 1.7451796566601843,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0015,
-      "reward": 5.246292740106583,
-      "reward_std": 2.9751971366640646,
-      "rewards/concensus_correctness_reward_func": 1.6888750046491623,
-      "rewards/consensus_reward_func": 0.875,
+      "loss": 0.0017,
+      "reward": 4.474540323019028,
+      "reward_std": 1.2630424201197457,
+      "rewards/concensus_correctness_reward_func": 1.2620625011622906,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.6931990003213286,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.6150090787559748,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.0985937491059303,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.1130937486886978,
       "step": 10
     },
     {
-      "completion_length": 183.625,
-      "epoch": 0.4873096446700508,
-      "grad_norm": 23.80231475830078,
-      "kl": 0.18446664046496153,
+      "completion_length": 206.59375,
+      "epoch": 0.5962732919254659,
+      "grad_norm": 45.16315841674805,
+      "kl": 2.844578029587865,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0002,
-      "reward": 3.904319927096367,
-      "reward_std": 0.8808909669169225,
-      "rewards/concensus_correctness_reward_func": 0.8669999986886978,
-      "rewards/consensus_reward_func": 0.6875,
+      "loss": 0.0028,
+      "reward": 3.3002478405833244,
+      "reward_std": 1.4689021888189018,
+      "rewards/concensus_correctness_reward_func": 0.6563750021159649,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6076324637979269,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5149665717035532,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.1171875,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.11328125,
       "step": 12
     },
     {
-      "completion_length": 197.9375,
-      "epoch": 0.5685279187817259,
-      "grad_norm": 29.56537628173828,
-      "kl": 0.335901414626278,
+      "completion_length": 199.78125,
+      "epoch": 0.6956521739130435,
+      "grad_norm": 703.441650390625,
+      "kl": 30.634997794870287,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0003,
-      "reward": 3.6152326092123985,
-      "reward_std": 2.132979861460626,
-      "rewards/concensus_correctness_reward_func": 0.9984999969601631,
+      "loss": 0.0306,
+      "reward": 4.00101051479578,
+      "reward_std": 2.141244651051238,
+      "rewards/concensus_correctness_reward_func": 1.3625624813139439,
       "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.6354825887829065,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5683229928836226,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 0.9656250029802322,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.1482499986886978,
       "step": 14
     },
     {
-      "completion_length": 188.5,
-      "epoch": 0.649746192893401,
-      "grad_norm": 18.8956298828125,
-      "kl": 0.1332387945149094,
+      "completion_length": 189.5625,
+      "epoch": 0.7950310559006211,
+      "grad_norm": 22.315275192260742,
+      "kl": 0.6838863966986537,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0001,
-      "reward": 5.6154617220163345,
-      "reward_std": 2.4665655323915416,
-      "rewards/concensus_correctness_reward_func": 2.249625027179718,
-      "rewards/consensus_reward_func": 1.0625,
+      "loss": 0.0007,
+      "reward": 5.960442878305912,
+      "reward_std": 1.6150732710957527,
+      "rewards/concensus_correctness_reward_func": 2.5391249880194664,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.6040242798626423,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5619429536163807,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.152437500655651,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.09375,
       "step": 16
     },
     {
-      "completion_length": 254.21875,
-      "epoch": 0.7309644670050761,
-      "grad_norm": 17.760560989379883,
-      "kl": 0.16061328526120633,
+      "completion_length": 197.75,
+      "epoch": 0.8944099378881988,
+      "grad_norm": 52.568763732910156,
+      "kl": 2.449893494369462,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0002,
-      "reward": 2.3994189221411943,
-      "reward_std": 0.6981450718012638,
-      "rewards/concensus_correctness_reward_func": 0.2383124940097332,
-      "rewards/consensus_reward_func": 0.375,
+      "loss": 0.0024,
+      "reward": 4.1809456795454025,
+      "reward_std": 2.145681773661636,
+      "rewards/concensus_correctness_reward_func": 1.2196249850094318,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.35088759183418006,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.582414451870136,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 0.9977187514305115,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.14453125,
       "step": 18
     },
     {
-      "completion_length": 209.71875,
-      "epoch": 0.8121827411167513,
-      "grad_norm": 37.05254364013672,
-      "kl": 0.19567961676511914,
+      "completion_length": 247.875,
+      "epoch": 0.9937888198757764,
+      "grad_norm": 106.65875244140625,
+      "kl": 6.712249951437116,
       "learning_rate": 0.0,
-      "loss": 0.0002,
-      "reward": 4.4470479413867,
-      "reward_std": 2.0779289572965354,
-      "rewards/concensus_correctness_reward_func": 1.0115625336766243,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.0067,
+      "reward": 4.293041463941336,
+      "reward_std": 1.9913227264769375,
+      "rewards/concensus_correctness_reward_func": 1.4836874902248383,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.6347042080014944,
+      "rewards/question_recreation_reward_func": 0.5593539979308844,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.11328125,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 0.984375,
       "step": 20
     },
     {
-      "epoch": 0.8121827411167513,
+      "epoch": 0.9937888198757764,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.00031505114329775095,
-      "train_runtime": 1048.8673,
-      "train_samples_per_second": 0.305,
-      "train_steps_per_second": 0.019
+      "train_loss": 0.00457829560036771,
+      "train_runtime": 805.5031,
+      "train_samples_per_second": 0.397,
+      "train_steps_per_second": 0.025
     }
   ],
   "logging_steps": 2,