End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.468176003973349,
-    "train_runtime": 141.5805,
-    "train_samples": 160,
-    "train_samples_per_second": 2.26,
-    "train_steps_per_second": 0.141
+    "train_loss": 2.8162464786873898e-06,
+    "train_runtime": 2270.13,
+    "train_samples": 320,
+    "train_samples_per_second": 0.141,
+    "train_steps_per_second": 0.009
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8bc89d5b0eef6c96f3eeb6f0589367b381fc1f8257b206c43e0fcecb2132fce9
+oid sha256:cc6f6749c834bc252a24de8d11fe3707d06262cd3b8f6d225f4f4a3c34624ee4
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.468176003973349,
-    "train_runtime": 141.5805,
-    "train_samples": 160,
-    "train_samples_per_second": 2.26,
-    "train_steps_per_second": 0.141
+    "train_loss": 2.8162464786873898e-06,
+    "train_runtime": 2270.13,
+    "train_samples": 320,
+    "train_samples_per_second": 0.141,
+    "train_steps_per_second": 0.009
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.5,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 238.21875,
-      "epoch": 0.1,
-      "grad_norm": 23.061830520629883,
+      "completion_length": 227.59375,
+      "epoch": 0.05,
+      "grad_norm": 25.247303009033203,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 4.5968284253031015,
-      "reward_std": 0.616630203621753,
-      "rewards/concensus_correctness_reward_func": 2.199562504887581,
-      "rewards/consensus_reward_func": 0.9375,
+      "reward": 0.43621697928756475,
+      "reward_std": 0.3773691598617006,
+      "rewards/concensus_correctness_reward_func": 0.057500001043081284,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5915159117430449,
+      "rewards/question_recreation_reward_func": 0.1438732324168086,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6807499742135406,
+      "rewards/xmlcount_reward_func": 0.04734375057159923,
       "step": 2
     },
     {
-      "completion_length": 165.375,
-      "epoch": 0.2,
-      "grad_norm": 28.605224609375,
-      "kl": 0.1424046578758862,
+      "completion_length": 403.96875,
+      "epoch": 0.1,
+      "grad_norm": 30.580951690673828,
+      "kl": 0.004594966503645992,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0001,
-      "reward": 5.776076227426529,
-      "reward_std": 0.6955992908915505,
-      "rewards/concensus_correctness_reward_func": 1.6770624928176403,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0,
+      "reward": 0.21478787978412583,
+      "reward_std": 0.23127917431702372,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9044200219213963,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.10060038062511012,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 1.1789687052369118,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.05168749950826168,
       "step": 4
     },
     {
-      "completion_length": 190.0,
-      "epoch": 0.3,
-      "grad_norm": 23.82851219177246,
-      "kl": 0.34406266862060875,
+      "completion_length": 416.21875,
+      "epoch": 0.15,
+      "grad_norm": 68.20472717285156,
+      "kl": 0.001207319201057544,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0003,
-      "reward": 5.5506651401519775,
-      "reward_std": 0.16025130550133326,
-      "rewards/concensus_correctness_reward_func": 1.6028124950826168,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0,
+      "reward": 0.31035696691833436,
+      "reward_std": 0.4425280693176319,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9063213616609573,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.14985696435905993,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1665311940014362,
+      "rewards/xmlcount_reward_func": 0.09800000162795186,
       "step": 6
     },
     {
-      "completion_length": 172.96875,
-      "epoch": 0.4,
-      "grad_norm": 33.95582580566406,
-      "kl": 0.27934576687403023,
+      "completion_length": 367.34375,
+      "epoch": 0.2,
+      "grad_norm": 7.403642654418945,
+      "kl": 0.0010949020979751367,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0003,
-      "reward": 6.021767497062683,
-      "reward_std": 0.3667241964285495,
-      "rewards/concensus_correctness_reward_func": 1.857812493108213,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0,
+      "reward": 0.42255881475284696,
+      "reward_std": 0.41626901473500766,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9293612577021122,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.17862132983282208,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 1.1564687080681324,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.11893750075250864,
       "step": 8
     },
     {
-      "completion_length": 153.75,
-      "epoch": 0.5,
-      "grad_norm": 18.014972686767578,
-      "kl": 2.704328676336445,
+      "completion_length": 316.3125,
+      "epoch": 0.25,
+      "grad_norm": 74.33928680419922,
+      "kl": 0.002202798434154829,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0027,
-      "reward": 6.106728702783585,
-      "reward_std": 0.3038245351635851,
-      "rewards/concensus_correctness_reward_func": 1.9768749848008156,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 0.0,
+      "reward": 0.3897631341824308,
+      "reward_std": 0.8568756857785047,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9085097648203373,
+      "rewards/question_recreation_reward_func": 0.13716937709250487,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1588436998426914,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.04946874629240483,
       "step": 10
     },
     {
-      "completion_length": 162.5,
-      "epoch": 0.6,
-      "grad_norm": 97.16446685791016,
-      "kl": 5.110307271126658,
+      "completion_length": 304.25,
+      "epoch": 0.3,
+      "grad_norm": 34.88124084472656,
+      "kl": 0.007154400140279904,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0051,
-      "reward": 5.1675363183021545,
-      "reward_std": 0.8226150240807328,
-      "rewards/concensus_correctness_reward_func": 1.5147499814629555,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 0.0,
+      "reward": 0.5133137404918671,
+      "reward_std": 0.5358899807324633,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7865362111479044,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.22212625079555437,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1787499599158764,
+      "rewards/xmlcount_reward_func": -0.02131250022648601,
       "step": 12
     },
     {
-      "completion_length": 153.4375,
-      "epoch": 0.7,
-      "grad_norm": 14.0081205368042,
-      "kl": 0.31082624034024775,
+      "completion_length": 219.625,
+      "epoch": 0.35,
+      "grad_norm": 127.43692016601562,
+      "kl": 0.004367175111838151,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0003,
-      "reward": 6.385396376252174,
-      "reward_std": 1.4748346143751405,
-      "rewards/concensus_correctness_reward_func": 2.3415624871850014,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0,
+      "reward": 0.3028914491296746,
+      "reward_std": 0.27629712063935585,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8933338597416878,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.16801644733641297,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.2129999436438084,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.11924999952316284,
       "step": 14
     },
     {
-      "completion_length": 155.375,
-      "epoch": 0.8,
-      "grad_norm": 14.671515464782715,
-      "kl": 9.717409254400991,
+      "completion_length": 183.9375,
+      "epoch": 0.4,
+      "grad_norm": 27.21989631652832,
+      "kl": 0.0024261638791358564,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0097,
-      "reward": 5.68576055765152,
-      "reward_std": 0.6367222345579648,
-      "rewards/concensus_correctness_reward_func": 1.6311874985694885,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0,
+      "reward": 0.4117564453044906,
+      "reward_std": 0.6205049808704644,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8846354763954878,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.12513145490083843,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 1.2168124467134476,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.03662499878555536,
       "step": 16
     },
     {
-      "completion_length": 159.9375,
-      "epoch": 0.9,
-      "grad_norm": 134988.875,
-      "kl": 4659.71151078766,
+      "completion_length": 306.71875,
+      "epoch": 0.45,
+      "grad_norm": 16.683961868286133,
+      "kl": 0.0033253404471906833,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 4.6597,
-      "reward": 7.275305807590485,
-      "reward_std": 0.2323648716119351,
-      "rewards/concensus_correctness_reward_func": 3.144124999642372,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0,
+      "reward": 0.4757386976853013,
+      "reward_std": 0.4481022479594685,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8945556059479713,
+      "rewards/question_recreation_reward_func": 0.17830120405415073,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1741249524056911,
+      "rewards/xmlcount_reward_func": 0.10993749974295497,
       "step": 18
     },
     {
-      "completion_length": 157.0625,
-      "epoch": 1.0,
-      "grad_norm": 35.6768684387207,
-      "kl": 3.506329826777801,
+      "completion_length": 426.46875,
+      "epoch": 0.5,
+      "grad_norm": 12.958433151245117,
+      "kl": 0.0013902449518354842,
       "learning_rate": 0.0,
-      "loss": 0.0035,
-      "reward": 5.915709361433983,
-      "reward_std": 0.2794380595878465,
-      "rewards/concensus_correctness_reward_func": 1.8087499924004078,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0,
+      "reward": 0.3283228897198569,
+      "reward_std": 0.4778002558341541,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8960218746215105,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.17729164237971418,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 1.1328124590218067,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.026031247864011675,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.5,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.468176003973349,
-      "train_runtime": 141.5805,
-      "train_samples_per_second": 2.26,
-      "train_steps_per_second": 0.141
+      "train_loss": 2.8162464786873898e-06,
+      "train_runtime": 2270.13,
+      "train_samples_per_second": 0.141,
+      "train_steps_per_second": 0.009
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:910906187abeb0a5f817ca9add5448647f8f3972958bd098b989b8cf31cb6580
+oid sha256:4aea63f9733333abb9deb1edddd10a317bfbd746c34b03ac6a703aacb7a8ec98
 size 6008