End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.489093321855762e-06,
-    "train_runtime": 278.934,
-    "train_samples": 160,
-    "train_samples_per_second": 1.147,
-    "train_steps_per_second": 0.072
+    "train_loss": 2.187862992286682e-06,
+    "train_runtime": 301.973,
+    "train_samples": 180,
+    "train_samples_per_second": 1.06,
+    "train_steps_per_second": 0.066
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f9ad9a9029317ad2f6aec42b44dc24b1d3b7a127aa0a2b71b9e851b52b1778b
+oid sha256:9b2bd4b225a00d17d012bae8562b69a791d09ba646612b41192b1abcae7033c5
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.489093321855762e-06,
-    "train_runtime": 278.934,
-    "train_samples": 160,
-    "train_samples_per_second": 1.147,
-    "train_steps_per_second": 0.072
+    "train_loss": 2.187862992286682e-06,
+    "train_runtime": 301.973,
+    "train_samples": 180,
+    "train_samples_per_second": 1.06,
+    "train_steps_per_second": 0.066
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.8888888888888888,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 386.53125,
-      "epoch": 0.1,
-      "grad_norm": 11.838155746459961,
+      "completion_length": 477.0,
+      "epoch": 0.08888888888888889,
+      "grad_norm": 34.471927642822266,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 0.49732657615095377,
-      "reward_std": 0.6507052630768158,
-      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.174295978154987,
+      "reward_std": 1.7542744056117954,
+      "rewards/concensus_correctness_reward_func": 0.8310000002384186,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.19863907300168648,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.17798347567440942,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1136874994263053,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.08718749997206032,
       "step": 2
     },
     {
-      "completion_length": 289.15625,
-      "epoch": 0.2,
-      "grad_norm": 18.39859390258789,
-      "kl": 0.0021505101231014123,
+      "completion_length": 277.25,
+      "epoch": 0.17777777777777778,
+      "grad_norm": 26.461362838745117,
+      "kl": 0.00166156359409797,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.19581820373423398,
-      "reward_std": 0.4207594294275623,
+      "reward": 0.6097803856246173,
+      "reward_std": 0.5241530751809478,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.20109945279546082,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.2592491409741342,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.0677812471985817,
+      "rewards/xmlcount_reward_func": 0.20990624744445086,
       "step": 4
     },
     {
-      "completion_length": 445.0625,
-      "epoch": 0.3,
-      "grad_norm": 15.898308753967285,
-      "kl": 0.0011719617286871653,
+      "completion_length": 386.34375,
+      "epoch": 0.26666666666666666,
+      "grad_norm": 11.892231941223145,
+      "kl": 0.0016316118271788582,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.1751822994556278,
-      "reward_std": 0.43424874570337124,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.9368757626507431,
+      "reward_std": 1.3130901722470298,
+      "rewards/concensus_correctness_reward_func": 0.1224374994635582,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.16821355387219228,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.2546570213744417,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.006968748988583684,
+      "rewards/xmlcount_reward_func": 0.12228124856483191,
       "step": 6
     },
     {
-      "completion_length": 325.625,
-      "epoch": 0.4,
-      "grad_norm": 155.9239959716797,
-      "kl": 0.010154961106309202,
+      "completion_length": 324.9375,
+      "epoch": 0.35555555555555557,
+      "grad_norm": 22.727386474609375,
+      "kl": 0.0020563425605359953,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.45496489957440645,
-      "reward_std": 0.7626551468565594,
-      "rewards/concensus_correctness_reward_func": 0.11950000002980232,
+      "reward": 2.560941856354475,
+      "reward_std": 1.6560740754939616,
+      "rewards/concensus_correctness_reward_func": 1.875,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.23943364236038178,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.258441973477602,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.028968748403713107,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.16187499649822712,
       "step": 8
     },
     {
-      "completion_length": 406.40625,
-      "epoch": 0.5,
-      "grad_norm": 16.150299072265625,
-      "kl": 0.0016187584624276496,
+      "completion_length": 466.5625,
+      "epoch": 0.4444444444444444,
+      "grad_norm": 4.7971391677856445,
+      "kl": 0.0010438499575684546,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.3459215434268117,
-      "reward_std": 0.6205616659717634,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.5703212053049356,
+      "reward_std": 0.8992374676745385,
+      "rewards/concensus_correctness_reward_func": 0.12262500077486038,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.15970279998145998,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.19388369610533118,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.0012812484055757523,
+      "rewards/xmlcount_reward_func": 0.06631250027567148,
       "step": 10
     },
     {
-      "completion_length": 402.1875,
-      "epoch": 0.6,
-      "grad_norm": 12.852953910827637,
-      "kl": 0.0021437756731756963,
+      "completion_length": 476.96875,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 10.955451011657715,
+      "kl": 0.0013942485475126887,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.7064267341047525,
-      "reward_std": 0.9020523639192106,
-      "rewards/concensus_correctness_reward_func": 0.17631249874830246,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.5070485147298314,
+      "reward_std": 0.39075206633424386,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.2865517425816506,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.198392266232986,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.056062495801597834,
+      "rewards/xmlcount_reward_func": 0.1211562491953373,
       "step": 12
     },
     {
-      "completion_length": 304.875,
-      "epoch": 0.7,
-      "grad_norm": 7.3621931076049805,
-      "kl": 0.002075214839351247,
+      "completion_length": 502.15625,
+      "epoch": 0.6222222222222222,
+      "grad_norm": 5.031079292297363,
+      "kl": 0.0016764367901487276,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.36785096395760775,
-      "reward_std": 0.46947701514000073,
+      "reward": 0.3395441100001335,
+      "reward_std": 0.43814076797571033,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.1592259633471258,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.1331691121449694,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.08362499932991341,
+      "rewards/xmlcount_reward_func": -0.10612499457783997,
       "step": 14
     },
     {
-      "completion_length": 288.5,
-      "epoch": 0.8,
-      "grad_norm": 21.77416229248047,
-      "kl": 0.002049503083981108,
+      "completion_length": 339.5,
+      "epoch": 0.7111111111111111,
+      "grad_norm": 10.80864143371582,
+      "kl": 0.0050469062916818075,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.5508579231100157,
-      "reward_std": 0.6642407482140698,
-      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "reward": 0.33586260332958773,
+      "reward_std": 0.4518825721988833,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.2362329208990559,
+      "rewards/question_recreation_reward_func": 0.1327063522185199,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.12949999934062362,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.06253124959766865,
       "step": 16
     },
     {
-      "completion_length": 434.53125,
-      "epoch": 0.9,
-      "grad_norm": 18.984968185424805,
-      "kl": 0.001922691324580228,
+      "completion_length": 362.3125,
+      "epoch": 0.8,
+      "grad_norm": 15.09556770324707,
+      "kl": 0.0029296423817868344,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.36621046916843625,
-      "reward_std": 0.6980189112837252,
-      "rewards/concensus_correctness_reward_func": 0.12012500315904617,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.0260260263457894,
+      "reward_std": 1.464539691573009,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.13721047036960954,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.24865100963506848,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.016125003341585398,
+      "rewards/xmlcount_reward_func": 0.02737499843351543,
       "step": 18
     },
     {
-      "completion_length": 406.34375,
-      "epoch": 1.0,
-      "grad_norm": 5.100473880767822,
-      "kl": 0.0013912481890656636,
+      "completion_length": 406.875,
+      "epoch": 0.8888888888888888,
+      "grad_norm": 73.40243530273438,
+      "kl": 0.006029042899172055,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.1821420846972615,
-      "reward_std": 1.2628907886974048,
-      "rewards/concensus_correctness_reward_func": 0.6661874987185001,
+      "reward": 1.030112552922219,
+      "reward_std": 1.194272859574994,
+      "rewards/concensus_correctness_reward_func": 0.625,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.20657960418611765,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.16539380967151374,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.0031250016763806343,
+      "rewards/xmlcount_reward_func": 0.11471875058487058,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.8888888888888888,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.489093321855762e-06,
-      "train_runtime": 278.934,
-      "train_samples_per_second": 1.147,
-      "train_steps_per_second": 0.072
+      "train_loss": 2.187862992286682e-06,
+      "train_runtime": 301.973,
+      "train_samples_per_second": 1.06,
+      "train_steps_per_second": 0.066
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f300602dac5237e6ccff2cd0bf6802e40d6e41c96b1b12e45ffeda8651d9e8e
+oid sha256:1b643556e8d9899398708fcc4b831391b60250ddc912ac1b5c0d2b1b3447e34e
 size 5944