End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0011644231490208767,
-    "train_runtime": 821.0873,
-    "train_samples": 160,
-    "train_samples_per_second": 0.39,
-    "train_steps_per_second": 0.024
+    "train_loss": 0.00031505114329775095,
+    "train_runtime": 1048.8673,
+    "train_samples": 197,
+    "train_samples_per_second": 0.305,
+    "train_steps_per_second": 0.019
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a2c3480832e3d044554cff4d456dd7b5e6641763c9574b1863e2fdf52b5004a
+oid sha256:2e197570708b7b13c3c45c47e6b280af2936842e7c55027621ebf5169ae627a0
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0011644231490208767,
-    "train_runtime": 821.0873,
-    "train_samples": 160,
-    "train_samples_per_second": 0.39,
-    "train_steps_per_second": 0.024
+    "train_loss": 0.00031505114329775095,
+    "train_runtime": 1048.8673,
+    "train_samples": 197,
+    "train_samples_per_second": 0.305,
+    "train_steps_per_second": 0.019
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.8121827411167513,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 268.5,
-      "epoch": 0.1,
-      "grad_norm": 19.180315017700195,
+      "completion_length": 347.0,
+      "epoch": 0.08121827411167512,
+      "grad_norm": 25.34670639038086,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
-      "loss": -0.0,
-      "reward": 2.194716626778245,
-      "reward_std": 1.0458471510210074,
-      "rewards/concensus_correctness_reward_func": 0.4673749953508377,
-      "rewards/consensus_reward_func": 0.25,
+      "loss": 0.0,
+      "reward": 1.8532088848296553,
+      "reward_std": 0.7197320554842008,
+      "rewards/concensus_correctness_reward_func": 0.42650000005960464,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.34868538519367576,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4665213600965217,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 0.6442812544992194,
+      "rewards/strict_format_reward_func": 0.109375,
+      "rewards/xmlcount_reward_func": 0.22581250546500087,
       "step": 2
     },
     {
-      "completion_length": 212.1875,
-      "epoch": 0.2,
-      "grad_norm": 17.529966354370117,
-      "kl": 0.019923070445656776,
+      "completion_length": 214.8125,
+      "epoch": 0.16243654822335024,
+      "grad_norm": 21.418426513671875,
+      "kl": 0.02307581211789511,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 4.613661300390959,
-      "reward_std": 1.688361722859554,
-      "rewards/concensus_correctness_reward_func": 1.4407500103116035,
+      "reward": 3.239914685487747,
+      "reward_std": 1.1554774387914222,
+      "rewards/concensus_correctness_reward_func": 0.45393750071525574,
       "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.7822863161563873,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.5888834362849593,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.09375,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.1033437550067902,
       "step": 4
     },
     {
-      "completion_length": 191.5,
-      "epoch": 0.3,
-      "grad_norm": 35.43332290649414,
-      "kl": 0.04699859209358692,
+      "completion_length": 190.75,
+      "epoch": 0.2436548223350254,
+      "grad_norm": 35.000457763671875,
+      "kl": 0.3683422572212294,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0,
-      "reward": 3.027126722037792,
-      "reward_std": 1.3399831917195115,
-      "rewards/concensus_correctness_reward_func": 0.5177500061690807,
-      "rewards/consensus_reward_func": 0.1875,
+      "loss": 0.0004,
+      "reward": 3.8262907788157463,
+      "reward_std": 1.3872474904637784,
+      "rewards/concensus_correctness_reward_func": 1.1668749935925007,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.5017204843461514,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.6158845135942101,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 1.0232812464237213,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.1060312539339066,
       "step": 6
     },
     {
-      "completion_length": 144.34375,
-      "epoch": 0.4,
-      "grad_norm": 33.76642608642578,
-      "kl": 1.42726071679499,
+      "completion_length": 202.8125,
+      "epoch": 0.3248730964467005,
+      "grad_norm": 16.695356369018555,
+      "kl": 0.21090720151551068,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0014,
-      "reward": 4.323728188872337,
-      "reward_std": 1.5747067951597273,
-      "rewards/concensus_correctness_reward_func": 1.0916874967515469,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.0002,
+      "reward": 3.1408818289637566,
+      "reward_std": 1.3075056065281387,
+      "rewards/concensus_correctness_reward_func": 0.2862499989569187,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5211031236685812,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6319755638251081,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.2109375,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 1.14453125,
       "step": 8
     },
     {
-      "completion_length": 139.25,
-      "epoch": 0.5,
-      "grad_norm": 29.404335021972656,
-      "kl": 0.3537763834465295,
+      "completion_length": 239.71875,
+      "epoch": 0.40609137055837563,
+      "grad_norm": 17.293676376342773,
+      "kl": 1.533459787024185,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0004,
-      "reward": 5.119718864560127,
-      "reward_std": 2.458271285984665,
-      "rewards/concensus_correctness_reward_func": 1.5305624641478062,
-      "rewards/consensus_reward_func": 0.5625,
+      "loss": 0.0015,
+      "reward": 5.246292740106583,
+      "reward_std": 2.9751971366640646,
+      "rewards/concensus_correctness_reward_func": 1.6888750046491623,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.6360313631594181,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.6931990003213286,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.21875,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 1.0985937491059303,
       "step": 10
     },
     {
-      "completion_length": 194.8125,
-      "epoch": 0.6,
-      "grad_norm": 20.139419555664062,
-      "kl": 0.21648985077627003,
+      "completion_length": 183.625,
+      "epoch": 0.4873096446700508,
+      "grad_norm": 23.80231475830078,
+      "kl": 0.18446664046496153,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0002,
-      "reward": 3.071705974638462,
-      "reward_std": 1.3920980363618582,
-      "rewards/concensus_correctness_reward_func": 0.5750625021755695,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 3.904319927096367,
+      "reward_std": 0.8808909669169225,
+      "rewards/concensus_correctness_reward_func": 0.8669999986886978,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6628309753723443,
+      "rewards/question_recreation_reward_func": 0.6076324637979269,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.0369375050067902,
+      "rewards/strict_format_reward_func": 0.375,
+      "rewards/xmlcount_reward_func": 1.1171875,
       "step": 12
     },
     {
-      "completion_length": 130.1875,
-      "epoch": 0.7,
-      "grad_norm": 131.01979064941406,
-      "kl": 8.531492722569965,
+      "completion_length": 197.9375,
+      "epoch": 0.5685279187817259,
+      "grad_norm": 29.56537628173828,
+      "kl": 0.335901414626278,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0085,
-      "reward": 3.87683779746294,
-      "reward_std": 1.9874035753309727,
-      "rewards/concensus_correctness_reward_func": 0.5249375011771917,
-      "rewards/consensus_reward_func": 0.625,
+      "loss": 0.0003,
+      "reward": 3.6152326092123985,
+      "reward_std": 2.132979861460626,
+      "rewards/concensus_correctness_reward_func": 0.9984999969601631,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.6565877520479262,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.6354825887829065,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.1015625,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 0.9656250029802322,
       "step": 14
     },
     {
-      "completion_length": 186.53125,
-      "epoch": 0.8,
-      "grad_norm": 21.889812469482422,
-      "kl": 0.2329507569083944,
+      "completion_length": 188.5,
+      "epoch": 0.649746192893401,
+      "grad_norm": 18.8956298828125,
+      "kl": 0.1332387945149094,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0002,
-      "reward": 4.2741429060697556,
-      "reward_std": 1.4219048471131828,
-      "rewards/concensus_correctness_reward_func": 1.0795624926686287,
-      "rewards/consensus_reward_func": 0.4375,
+      "loss": 0.0001,
+      "reward": 5.6154617220163345,
+      "reward_std": 2.4665655323915416,
+      "rewards/concensus_correctness_reward_func": 2.249625027179718,
+      "rewards/consensus_reward_func": 1.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.6366117261350155,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.6040242798626423,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.2142187505960464,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.152437500655651,
       "step": 16
     },
     {
-      "completion_length": 161.15625,
-      "epoch": 0.9,
-      "grad_norm": 25.868162155151367,
-      "kl": 0.45650642761029303,
+      "completion_length": 254.21875,
+      "epoch": 0.7309644670050761,
+      "grad_norm": 17.760560989379883,
+      "kl": 0.16061328526120633,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0005,
-      "reward": 3.4033926129341125,
-      "reward_std": 1.403730297461152,
-      "rewards/concensus_correctness_reward_func": 0.43843749538064003,
-      "rewards/consensus_reward_func": 0.1875,
+      "loss": 0.0002,
+      "reward": 2.3994189221411943,
+      "reward_std": 0.6981450718012638,
+      "rewards/concensus_correctness_reward_func": 0.2383124940097332,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.5665176380425692,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.35088759183418006,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.1328125,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 0.9977187514305115,
       "step": 18
     },
     {
-      "completion_length": 136.40625,
-      "epoch": 1.0,
-      "grad_norm": 18.451465606689453,
-      "kl": 0.3661159457406029,
+      "completion_length": 209.71875,
+      "epoch": 0.8121827411167513,
+      "grad_norm": 37.05254364013672,
+      "kl": 0.19567961676511914,
       "learning_rate": 0.0,
-      "loss": 0.0004,
-      "reward": 4.148729115724564,
-      "reward_std": 2.1622865115641616,
-      "rewards/concensus_correctness_reward_func": 1.2454375103116035,
-      "rewards/consensus_reward_func": 0.375,
+      "loss": 0.0002,
+      "reward": 4.4470479413867,
+      "reward_std": 2.0779289572965354,
+      "rewards/concensus_correctness_reward_func": 1.0115625336766243,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.696229110006243,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.6347042080014944,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.1758125014603138,
+      "rewards/strict_format_reward_func": 0.375,
+      "rewards/xmlcount_reward_func": 1.11328125,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.8121827411167513,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.0011644231490208767,
-      "train_runtime": 821.0873,
-      "train_samples_per_second": 0.39,
-      "train_steps_per_second": 0.024
+      "train_loss": 0.00031505114329775095,
+      "train_runtime": 1048.8673,
+      "train_samples_per_second": 0.305,
+      "train_steps_per_second": 0.019
     }
   ],
   "logging_steps": 2,