End of training

README.md

@@ -31,7 +31,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
- 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/haruseka2014-bitcoin-none/huggingface/runs/6vj92dqm) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8d5cd32c238d5128f1669c54131ccdd2d7efb5b7dd3a408098c20df45165f7e
+oid sha256:0368f9b6acabb1bfa084b64ea268c0886d43dbdeb1198c68defed6152e3c21e9
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.507646330395801e-07,
-    "train_runtime": 324.7295,
+    "train_loss": 6.933692645816336e-07,
+    "train_runtime": 335.7519,
     "train_samples": 14,
-    "train_samples_per_second": 0.985,
-    "train_steps_per_second": 0.062
+    "train_samples_per_second": 0.953,
+    "train_steps_per_second": 0.06
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.507646330395801e-07,
-    "train_runtime": 324.7295,
+    "train_loss": 6.933692645816336e-07,
+    "train_runtime": 335.7519,
     "train_samples": 14,
-    "train_samples_per_second": 0.985,
-    "train_steps_per_second": 0.062
+    "train_samples_per_second": 0.953,
+    "train_steps_per_second": 0.06
 }

trainer_state.json

@@ -9,203 +9,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 307.84375,
+      "completion_length": 373.09375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.47697943449020386,
-      "kl": 0.0007509570168622304,
+      "grad_norm": 0.373384565114975,
+      "kl": 0.0007221863561426289,
       "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 1.2545952871441841,
-      "reward_std": 1.0445544943213463,
-      "rewards/concensus_correctness_reward_func": 0.04481250047683716,
+      "reward": 2.6916642151772976,
+      "reward_std": 2.4385265596210957,
+      "rewards/concensus_correctness_reward_func": 1.3489374774508178,
       "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4747202843427658,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.4614454321563244,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.34443750232458115,
+      "rewards/xmlcount_reward_func": 0.2562812529504299,
       "step": 2
     },
     {
-      "completion_length": 283.7916666666667,
+      "completion_length": 269.9166666666667,
       "epoch": 1.0,
-      "grad_norm": 0.4536288380622864,
-      "kl": 0.000830039381980896,
+      "grad_norm": 0.33436840772628784,
+      "kl": 0.0009179994134077182,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 3.9119001924991608,
-      "reward_std": 4.873037169377009,
-      "rewards/concensus_correctness_reward_func": 2.0474166671435037,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 2.193864127000173,
+      "reward_std": 2.685385833183924,
+      "rewards/concensus_correctness_reward_func": 0.8278333333631357,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.46273347238699597,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.5319057752688726,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.235083336631457,
+      "rewards/xmlcount_reward_func": 0.3341249997417132,
       "step": 4
     },
     {
-      "completion_length": 311.71875,
+      "completion_length": 302.9375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.68195641040802,
-      "kl": 0.0008154660972650163,
+      "grad_norm": 0.4204976260662079,
+      "kl": 0.0007943223754409701,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 2.5976146487519145,
-      "reward_std": 2.239759039133787,
-      "rewards/concensus_correctness_reward_func": 1.302437499165535,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.5970541313290596,
+      "reward_std": 2.152347058057785,
+      "rewards/concensus_correctness_reward_func": 1.3126874770969152,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.37242713011801243,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.4615228455513716,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.36024999991059303,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.30721876211464405,
       "step": 6
     },
     {
-      "completion_length": 286.9166666666667,
+      "completion_length": 324.9166666666667,
       "epoch": 2.0,
-      "grad_norm": 0.3821156620979309,
-      "kl": 0.0008516140417971959,
+      "grad_norm": 0.369188129901886,
+      "kl": 0.0007666643941774964,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 4.207462310791016,
-      "reward_std": 2.887537010014057,
-      "rewards/concensus_correctness_reward_func": 1.8768334239721298,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 1.7628137369950612,
+      "reward_std": 1.1092421039938927,
+      "rewards/concensus_correctness_reward_func": 0.13908333331346512,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.5367539475361506,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.4734387770295143,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3772083173195521,
+      "rewards/xmlcount_reward_func": 0.40029167135556537,
       "step": 8
     },
     {
-      "completion_length": 296.90625,
+      "completion_length": 314.28125,
       "epoch": 2.571428571428571,
-      "grad_norm": 0.5019404888153076,
-      "kl": 0.0008093566793831997,
+      "grad_norm": 0.5270184278488159,
+      "kl": 0.0008258070301963016,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 2.2707934752106667,
-      "reward_std": 2.4034877978265285,
-      "rewards/concensus_correctness_reward_func": 0.7142499992623925,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.5355343967676163,
+      "reward_std": 0.9545740447938442,
+      "rewards/concensus_correctness_reward_func": 0.03987499978393316,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.43176219426095486,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.463753167539835,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.31228123791515827,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.39128125715069473,
       "step": 10
     },
     {
-      "completion_length": 300.4166666666667,
+      "completion_length": 301.75,
       "epoch": 3.0,
-      "grad_norm": 0.35668668150901794,
-      "kl": 0.0008771180194647362,
+      "grad_norm": 0.47715187072753906,
+      "kl": 0.0008599854190833867,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 1.604244440793991,
-      "reward_std": 1.1329847822586696,
-      "rewards/concensus_correctness_reward_func": 0.4125833222642541,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 3.0806438326835632,
+      "reward_std": 2.9016336326797805,
+      "rewards/concensus_correctness_reward_func": 1.68900000055631,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.462744377553463,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.4471021369099617,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.145583334689339,
+      "rewards/xmlcount_reward_func": 0.19454167348643145,
       "step": 12
     },
     {
-      "completion_length": 339.65625,
+      "completion_length": 335.53125,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.48297348618507385,
-      "kl": 0.0007411040933220647,
+      "grad_norm": 0.46307072043418884,
+      "kl": 0.0007349779334617779,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 3.51069688051939,
-      "reward_std": 3.406370159238577,
-      "rewards/concensus_correctness_reward_func": 1.5621250122785568,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 1.0974042024463415,
+      "reward_std": 1.304825820028782,
+      "rewards/concensus_correctness_reward_func": 0.046875,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.664228143170476,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5664042308926582,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.2530937436968088,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.15599999763071537,
       "step": 14
     },
     {
-      "completion_length": 309.3333333333333,
+      "completion_length": 348.4583333333333,
       "epoch": 4.0,
-      "grad_norm": 0.23572970926761627,
-      "kl": 0.0007775598302638779,
+      "grad_norm": 0.412990003824234,
+      "kl": 0.0007793560313681761,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 1.7949834987521172,
-      "reward_std": 2.426940143108368,
-      "rewards/concensus_correctness_reward_func": 0.8333333333333334,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 2.68368598818779,
+      "reward_std": 2.7279448012510934,
+      "rewards/concensus_correctness_reward_func": 1.1090832731376092,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.3045668974518776,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.35926927874485654,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3237500103811423,
+      "rewards/xmlcount_reward_func": 0.2153333487610022,
       "step": 16
     },
     {
-      "completion_length": 297.15625,
+      "completion_length": 367.5,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.7020504474639893,
-      "kl": 0.0008789005078142509,
+      "grad_norm": 0.6593879461288452,
+      "kl": 0.0007980476511875167,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 2.893677331507206,
-      "reward_std": 2.640741240233183,
-      "rewards/concensus_correctness_reward_func": 1.3593749990686774,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 2.2810916882008314,
+      "reward_std": 2.206418065354228,
+      "rewards/concensus_correctness_reward_func": 0.648250000551343,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5348647553473711,
+      "rewards/question_recreation_reward_func": 0.6310916803777218,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.37443749979138374,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4236249942332506,
       "step": 18
     },
     {
-      "completion_length": 261.875,
+      "completion_length": 324.0,
       "epoch": 5.0,
-      "grad_norm": 0.7190520763397217,
-      "kl": 0.001175893092295155,
+      "grad_norm": 0.4096703827381134,
+      "kl": 0.000849412016881009,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 1.0453936755657196,
-      "reward_std": 0.862831545372804,
-      "rewards/concensus_correctness_reward_func": 0.09633333484331767,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 1.0385125974814098,
+      "reward_std": 1.064698576927185,
+      "rewards/concensus_correctness_reward_func": 0.0551666667064031,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3342686705291271,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.4515126248200734,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.2606250022848447,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.19850001111626625,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.507646330395801e-07,
-      "train_runtime": 324.7295,
-      "train_samples_per_second": 0.985,
-      "train_steps_per_second": 0.062
+      "train_loss": 6.933692645816336e-07,
+      "train_runtime": 335.7519,
+      "train_samples_per_second": 0.953,
+      "train_steps_per_second": 0.06
     }
   ],
   "logging_steps": 2,