End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 60.04070474899636,
-    "train_runtime": 895.7192,
-    "train_samples": 222,
-    "train_samples_per_second": 0.357,
-    "train_steps_per_second": 0.022
+    "train_loss": 0.0011644231490208767,
+    "train_runtime": 821.0873,
+    "train_samples": 160,
+    "train_samples_per_second": 0.39,
+    "train_steps_per_second": 0.024
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e743448426b85677b8bdc90c6e2e258ea6aeac34ad52d250d679f312cc64e7b8
+oid sha256:0a2c3480832e3d044554cff4d456dd7b5e6641763c9574b1863e2fdf52b5004a
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 60.04070474899636,
-    "train_runtime": 895.7192,
-    "train_samples": 222,
-    "train_samples_per_second": 0.357,
-    "train_steps_per_second": 0.022
+    "train_loss": 0.0011644231490208767,
+    "train_runtime": 821.0873,
+    "train_samples": 160,
+    "train_samples_per_second": 0.39,
+    "train_steps_per_second": 0.024
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.7207207207207207,
+  "epoch": 1.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 228.59375,
-      "epoch": 0.07207207207207207,
-      "grad_norm": 33.895172119140625,
+      "completion_length": 268.5,
+      "epoch": 0.1,
+      "grad_norm": 19.180315017700195,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": -0.0,
-      "reward": 1.9609859569463879,
-      "reward_std": 0.8656060772482306,
-      "rewards/concensus_correctness_reward_func": 0.2421875,
+      "reward": 2.194716626778245,
+      "reward_std": 1.0458471510210074,
+      "rewards/concensus_correctness_reward_func": 0.4673749953508377,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.4285172347445041,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.34868538519367576,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.15625,
-      "rewards/xmlcount_reward_func": 0.6340312492102385,
+      "rewards/strict_format_reward_func": 0.171875,
+      "rewards/xmlcount_reward_func": 0.6442812544992194,
       "step": 2
     },
     {
-      "completion_length": 141.78125,
-      "epoch": 0.14414414414414414,
-      "grad_norm": 26.688365936279297,
-      "kl": 0.07244871955481358,
+      "completion_length": 212.1875,
+      "epoch": 0.2,
+      "grad_norm": 17.529966354370117,
+      "kl": 0.019923070445656776,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0001,
-      "reward": 4.899827718734741,
-      "reward_std": 2.5617886996478774,
-      "rewards/concensus_correctness_reward_func": 1.574062503874302,
-      "rewards/consensus_reward_func": 0.5625,
+      "loss": 0.0,
+      "reward": 4.613661300390959,
+      "reward_std": 1.688361722859554,
+      "rewards/concensus_correctness_reward_func": 1.4407500103116035,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.5406089581083506,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.7822863161563873,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.17578125,
+      "rewards/xmlcount_reward_func": 1.09375,
       "step": 4
     },
     {
-      "completion_length": 128.0,
-      "epoch": 0.21621621621621623,
-      "grad_norm": 24.55901527404785,
-      "kl": 0.18228621094021946,
+      "completion_length": 191.5,
+      "epoch": 0.3,
+      "grad_norm": 35.43332290649414,
+      "kl": 0.04699859209358692,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0002,
-      "reward": 3.3227058574557304,
-      "reward_std": 0.40170515893260017,
-      "rewards/concensus_correctness_reward_func": 0.4568124897778034,
-      "rewards/consensus_reward_func": 0.4375,
+      "loss": 0.0,
+      "reward": 3.027126722037792,
+      "reward_std": 1.3399831917195115,
+      "rewards/concensus_correctness_reward_func": 0.5177500061690807,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5533933327533305,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5017204843461514,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.21875,
+      "rewards/strict_format_reward_func": 0.171875,
+      "rewards/xmlcount_reward_func": 1.0232812464237213,
       "step": 6
     },
     {
-      "completion_length": 154.46875,
-      "epoch": 0.2882882882882883,
-      "grad_norm": 21.715429306030273,
-      "kl": 600388.9485286925,
+      "completion_length": 144.34375,
+      "epoch": 0.4,
+      "grad_norm": 33.76642608642578,
+      "kl": 1.42726071679499,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 600.389,
-      "reward": 3.769036091864109,
-      "reward_std": 0.9345146588166244,
-      "rewards/concensus_correctness_reward_func": 0.54356250166893,
-      "rewards/consensus_reward_func": 0.375,
+      "loss": 0.0014,
+      "reward": 4.323728188872337,
+      "reward_std": 1.5747067951597273,
+      "rewards/concensus_correctness_reward_func": 1.0916874967515469,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.7567235939204693,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5211031236685812,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.15625,
+      "rewards/strict_format_reward_func": 0.4375,
+      "rewards/xmlcount_reward_func": 1.2109375,
       "step": 8
     },
     {
-      "completion_length": 143.65625,
-      "epoch": 0.36036036036036034,
-      "grad_norm": 36.61812210083008,
-      "kl": 2.4715459311846644,
+      "completion_length": 139.25,
+      "epoch": 0.5,
+      "grad_norm": 29.404335021972656,
+      "kl": 0.3537763834465295,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0025,
-      "reward": 3.961671121418476,
-      "reward_std": 1.9286869522184134,
-      "rewards/concensus_correctness_reward_func": 0.6065624989569187,
-      "rewards/consensus_reward_func": 0.4375,
+      "loss": 0.0004,
+      "reward": 5.119718864560127,
+      "reward_std": 2.458271285984665,
+      "rewards/concensus_correctness_reward_func": 1.5305624641478062,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.7301086485385895,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.6360313631594181,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.1875,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.21875,
       "step": 10
     },
     {
-      "completion_length": 138.46875,
-      "epoch": 0.43243243243243246,
-      "grad_norm": 23.32555389404297,
-      "kl": 1.8817484425380826,
+      "completion_length": 194.8125,
+      "epoch": 0.6,
+      "grad_norm": 20.139419555664062,
+      "kl": 0.21648985077627003,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0019,
-      "reward": 4.619454130530357,
-      "reward_std": 1.5896258265711367,
-      "rewards/concensus_correctness_reward_func": 0.9133125059306622,
-      "rewards/consensus_reward_func": 0.4375,
+      "loss": 0.0002,
+      "reward": 3.071705974638462,
+      "reward_std": 1.3920980363618582,
+      "rewards/concensus_correctness_reward_func": 0.5750625021755695,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9375,
-      "rewards/question_recreation_reward_func": 0.6709854220971465,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6628309753723443,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.22265625,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.0369375050067902,
       "step": 12
     },
     {
-      "completion_length": 153.4375,
-      "epoch": 0.5045045045045045,
-      "grad_norm": 25.049509048461914,
-      "kl": 4.15379779948853,
+      "completion_length": 130.1875,
+      "epoch": 0.7,
+      "grad_norm": 131.01979064941406,
+      "kl": 8.531492722569965,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0042,
-      "reward": 4.069915443658829,
-      "reward_std": 1.2653860269347206,
-      "rewards/concensus_correctness_reward_func": 0.6948125064373016,
-      "rewards/consensus_reward_func": 0.4375,
+      "loss": 0.0085,
+      "reward": 3.87683779746294,
+      "reward_std": 1.9874035753309727,
+      "rewards/concensus_correctness_reward_func": 0.5249375011771917,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.650665394961834,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.6565877520479262,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.2088125012814999,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.1015625,
       "step": 14
     },
     {
-      "completion_length": 127.34375,
-      "epoch": 0.5765765765765766,
-      "grad_norm": 23.733936309814453,
-      "kl": 0.697271732846275,
+      "completion_length": 186.53125,
+      "epoch": 0.8,
+      "grad_norm": 21.889812469482422,
+      "kl": 0.2329507569083944,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0007,
-      "reward": 3.8320476040244102,
-      "reward_std": 1.3547778044012375,
-      "rewards/concensus_correctness_reward_func": 0.7881875038146973,
-      "rewards/consensus_reward_func": 0.5,
+      "loss": 0.0002,
+      "reward": 4.2741429060697556,
+      "reward_std": 1.4219048471131828,
+      "rewards/concensus_correctness_reward_func": 1.0795624926686287,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5712038641795516,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.6366117261350155,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.20703125,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.2142187505960464,
       "step": 16
     },
     {
-      "completion_length": 121.625,
-      "epoch": 0.6486486486486487,
-      "grad_norm": 72.80143737792969,
-      "kl": 1.9725859742611647,
+      "completion_length": 161.15625,
+      "epoch": 0.9,
+      "grad_norm": 25.868162155151367,
+      "kl": 0.45650642761029303,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0019,
-      "reward": 3.1732209473848343,
-      "reward_std": 0.9505836870034727,
-      "rewards/concensus_correctness_reward_func": 0.4818749949336052,
-      "rewards/consensus_reward_func": 0.4375,
+      "loss": 0.0005,
+      "reward": 3.4033926129341125,
+      "reward_std": 1.403730297461152,
+      "rewards/concensus_correctness_reward_func": 0.43843749538064003,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5605334475403652,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.5665176380425692,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.421875,
-      "rewards/xmlcount_reward_func": 1.2089374959468842,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.1328125,
       "step": 18
     },
     {
-      "completion_length": 126.125,
-      "epoch": 0.7207207207207207,
-      "grad_norm": 20.80611228942871,
-      "kl": 6.6607602457515895,
+      "completion_length": 136.40625,
+      "epoch": 1.0,
+      "grad_norm": 18.451465606689453,
+      "kl": 0.3661159457406029,
       "learning_rate": 0.0,
-      "loss": 0.0067,
-      "reward": 3.751609832048416,
-      "reward_std": 1.4225790165364742,
-      "rewards/concensus_correctness_reward_func": 0.41862500086426735,
-      "rewards/consensus_reward_func": 0.5625,
+      "loss": 0.0004,
+      "reward": 4.148729115724564,
+      "reward_std": 2.1622865115641616,
+      "rewards/concensus_correctness_reward_func": 1.2454375103116035,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.6689222925342619,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.696229110006243,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.421875,
-      "rewards/xmlcount_reward_func": 1.1796875,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.1758125014603138,
       "step": 20
     },
     {
-      "epoch": 0.7207207207207207,
+      "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 60.04070474899636,
-      "train_runtime": 895.7192,
-      "train_samples_per_second": 0.357,
-      "train_steps_per_second": 0.022
+      "train_loss": 0.0011644231490208767,
+      "train_runtime": 821.0873,
+      "train_samples_per_second": 0.39,
+      "train_steps_per_second": 0.024
     }
   ],
   "logging_steps": 2,