End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 20766.248711165834,
-    "train_runtime": 102.6386,
-    "train_samples": 28,
-    "train_samples_per_second": 3.118,
-    "train_steps_per_second": 0.195
+    "train_loss": 8.484272620989941e-06,
+    "train_runtime": 988.2843,
+    "train_samples": 63,
+    "train_samples_per_second": 0.324,
+    "train_steps_per_second": 0.02
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3319c4514a75c71a9635812cac658b8cdee638c133e5ea4d8b4da940a081af1b
+oid sha256:a2fc2dca7db3ecf1d1994b1d7023d3c4542bf9dba6a76bf8abff332ab7a9bf25
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 20766.248711165834,
-    "train_runtime": 102.6386,
-    "train_samples": 28,
-    "train_samples_per_second": 3.118,
-    "train_steps_per_second": 0.195
+    "train_loss": 8.484272620989941e-06,
+    "train_runtime": 988.2843,
+    "train_samples": 63,
+    "train_samples_per_second": 0.324,
+    "train_steps_per_second": 0.02
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 2.507936507936508,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 246.53125,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 62.44981002807617,
+      "completion_length": 328.78125,
+      "epoch": 0.25396825396825395,
+      "grad_norm": 17.286907196044922,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 2.81018253415823,
-      "reward_std": 1.405245827510953,
-      "rewards/concensus_correctness_reward_func": 0.5839375033974648,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 1.1846126343589276,
+      "reward_std": 1.602128185390029,
+      "rewards/concensus_correctness_reward_func": 0.6851250007748604,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5905575299984775,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.09375,
-      "rewards/xmlcount_reward_func": 0.7919374946504831,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.21126889549486805,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.022593744564801455,
       "step": 2
     },
     {
-      "completion_length": 126.41666666666667,
-      "epoch": 1.0,
-      "grad_norm": 36.17685317993164,
-      "kl": 5.282945941512783,
+      "completion_length": 368.875,
+      "epoch": 0.5079365079365079,
+      "grad_norm": 8.031682014465332,
+      "kl": 0.004274788287148112,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.004,
-      "reward": 4.323091174165408,
-      "reward_std": 1.6219139602035284,
-      "rewards/concensus_correctness_reward_func": 1.2741666634877522,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "loss": 0.0,
+      "reward": 1.8104806628543884,
+      "reward_std": 2.1033532274886966,
+      "rewards/concensus_correctness_reward_func": 0.8671249970793724,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5833333333333334,
-      "rewards/question_recreation_reward_func": 0.642924482623736,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3325744161847979,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 0.906000000735124,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.23578124959021807,
       "step": 4
     },
     {
-      "completion_length": 144.71875,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 21567.345703125,
-      "kl": 207468117.82215685,
+      "completion_length": 277.90625,
+      "epoch": 0.7619047619047619,
+      "grad_norm": 19.59160804748535,
+      "kl": 0.0013730914943153039,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 207468.1094,
-      "reward": 5.788680654019117,
-      "reward_std": 0.7503121838381048,
-      "rewards/concensus_correctness_reward_func": 1.78218749538064,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0,
+      "reward": 1.3974248827435076,
+      "reward_std": 1.4865058151553967,
+      "rewards/concensus_correctness_reward_func": 0.7475000023841858,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.8119618319906294,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.21223744517192245,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.085156250745058,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.18768750643357635,
       "step": 6
     },
     {
-      "completion_length": 143.625,
-      "epoch": 2.0,
-      "grad_norm": 151.89219665527344,
-      "kl": 48.836665123701096,
+      "completion_length": 399.8,
+      "epoch": 1.0,
+      "grad_norm": 216.4717559814453,
+      "kl": 0.0025436665513552726,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0366,
-      "reward": 5.109239861369133,
-      "reward_std": 0.582389307130749,
-      "rewards/concensus_correctness_reward_func": 1.4567500005165737,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0,
+      "reward": 0.6824912115931511,
+      "reward_std": 0.8270894133796295,
+      "rewards/concensus_correctness_reward_func": 0.057733333110809325,
+      "rewards/consensus_reward_func": 0.13333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7562398855419209,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3333333333333333,
-      "rewards/xmlcount_reward_func": 1.0629166662693024,
+      "rewards/final_correctness_reward_func": 0.06666666666666667,
+      "rewards/question_recreation_reward_func": 0.3987245598807931,
+      "rewards/soft_format_reward_func": 0.016666666666666666,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.009366654107968013,
       "step": 8
     },
     {
-      "completion_length": 133.9375,
-      "epoch": 2.571428571428571,
-      "grad_norm": 6474828.5,
-      "kl": 194250.27258606674,
+      "completion_length": 345.6875,
+      "epoch": 1.253968253968254,
+      "grad_norm": 23.902658462524414,
+      "kl": 0.0026058350231323857,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 194.2503,
-      "reward": 4.8971958719193935,
-      "reward_std": 0.6572542649810202,
-      "rewards/concensus_correctness_reward_func": 1.4178124964237213,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 0.7514822297962382,
+      "reward_std": 0.6706290168804117,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.7073208371875808,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.053312499076128,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.42210722726304084,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.18874999950639904,
       "step": 10
     },
     {
-      "completion_length": 131.79166666666666,
-      "epoch": 3.0,
-      "grad_norm": 63.40531921386719,
-      "kl": 46.34671560302377,
+      "completion_length": 230.28125,
+      "epoch": 1.507936507936508,
+      "grad_norm": 21.800233840942383,
+      "kl": 0.033992309923633,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0348,
-      "reward": 5.006111880143483,
-      "reward_std": 0.8839096422307193,
-      "rewards/concensus_correctness_reward_func": 1.4060000032186508,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "loss": 0.0,
+      "reward": 0.7290582247078419,
+      "reward_std": 0.9098360606294591,
+      "rewards/concensus_correctness_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.7846952006220818,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3065269637736492,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.0029166663686435,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.11003125557908788,
       "step": 12
     },
     {
-      "completion_length": 145.5625,
-      "epoch": 3.571428571428571,
-      "grad_norm": 49.8672981262207,
-      "kl": 9.690117788501084,
+      "completion_length": 407.34375,
+      "epoch": 1.7619047619047619,
+      "grad_norm": 64.31951141357422,
+      "kl": 0.001744390043313615,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0097,
-      "reward": 5.000209610909224,
-      "reward_std": 1.2152829460246721,
-      "rewards/concensus_correctness_reward_func": 1.3089375048875809,
-      "rewards/consensus_reward_func": 1.1875,
+      "loss": 0.0,
+      "reward": 0.5753998922882602,
+      "reward_std": 0.4874745855049696,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.7728346046060324,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.27893113857135177,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.04343749769032,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2339687505736947,
       "step": 14
     },
     {
-      "completion_length": 120.5,
-      "epoch": 4.0,
-      "grad_norm": 20.376087188720703,
-      "kl": 4.444862203362088,
+      "completion_length": 259.93333333333334,
+      "epoch": 2.0,
+      "grad_norm": 21.777584075927734,
+      "kl": 0.0022329573752358557,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0033,
-      "reward": 4.583630581696828,
-      "reward_std": 0.6420133346691728,
-      "rewards/concensus_correctness_reward_func": 1.330999990304311,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0,
+      "reward": 1.5026665093998115,
+      "reward_std": 1.081801247689873,
+      "rewards/concensus_correctness_reward_func": 0.7970666885375977,
+      "rewards/consensus_reward_func": 0.13333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.7358388638434311,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2916666666666667,
-      "rewards/xmlcount_reward_func": 1.0584583307305973,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2603665338208278,
+      "rewards/soft_format_reward_func": 0.016666666666666666,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.29523334639767806,
       "step": 16
     },
     {
-      "completion_length": 122.59375,
-      "epoch": 4.571428571428571,
-      "grad_norm": 57.4793701171875,
-      "kl": 5.2552297483198345,
+      "completion_length": 278.3125,
+      "epoch": 2.253968253968254,
+      "grad_norm": 76.7491455078125,
+      "kl": 0.010657810475095175,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0052,
-      "reward": 4.917819622904062,
-      "reward_std": 0.8335820628053625,
-      "rewards/concensus_correctness_reward_func": 1.4153125025331974,
-      "rewards/consensus_reward_func": 1.1875,
+      "loss": 0.0,
+      "reward": 1.249540930846706,
+      "reward_std": 1.0496195774467196,
+      "rewards/concensus_correctness_reward_func": 0.17406249791383743,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.7026322261663154,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.0030000023543835,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.47557220014277846,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2092812485061586,
       "step": 18
     },
     {
-      "completion_length": 156.41666666666666,
-      "epoch": 5.0,
-      "grad_norm": 20.131155014038086,
-      "kl": 45.09713823503504,
+      "completion_length": 314.3125,
+      "epoch": 2.507936507936508,
+      "grad_norm": 151.8843536376953,
+      "kl": 0.0257824770269508,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0338,
-      "reward": 4.907328523695469,
-      "reward_std": 0.8428607982350513,
-      "rewards/concensus_correctness_reward_func": 1.4174166570107143,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "loss": 0.0,
+      "reward": 1.567689070594497,
+      "reward_std": 1.5514863879216136,
+      "rewards/concensus_correctness_reward_func": 0.7475000023841858,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.7959951267888149,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.16115779627580196,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2708333333333333,
-      "rewards/xmlcount_reward_func": 1.089749999344349,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.20590625330805779,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 2.507936507936508,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 20766.248711165834,
-      "train_runtime": 102.6386,
-      "train_samples_per_second": 3.118,
-      "train_steps_per_second": 0.195
+      "train_loss": 8.484272620989941e-06,
+      "train_runtime": 988.2843,
+      "train_samples_per_second": 0.324,
+      "train_steps_per_second": 0.02
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 3,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4178419bb93870fd946cdbac7af5cdf92b06f0405898de966cb5649401ee70f1
+oid sha256:15a35ed0dc1b427b3d92ea468297276e7e5514301a2a4c403cab4aa16463d2e6
 size 5944