End of training

README.md

@@ -38,7 +38,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.50.3
+- Transformers: 4.51.3
 - Pytorch: 2.5.1
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.924667835235596e-05,
-    "train_runtime": 5955.7987,
-    "train_samples": 160,
-    "train_samples_per_second": 0.054,
-    "train_steps_per_second": 0.003
+    "train_loss": 1.61745585501194e-05,
+    "train_runtime": 5416.7432,
+    "train_samples": 28,
+    "train_samples_per_second": 0.059,
+    "train_steps_per_second": 0.004
 }

config.json

@@ -21,7 +21,7 @@
   "sliding_window": 32768,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.50.3",
+  "transformers_version": "4.51.3",
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936

generation_config.json

@@ -10,5 +10,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.50.3"
+  "transformers_version": "4.51.3"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b7a6fca9432ff7de0b07a80644bc1c6d54b903093eb9c689ba98effbb6e27054
+oid sha256:0e4f141e63038c991a13ded81acc0f4c23d0c88e4e2d06251a2cd2cdc33b72ba
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.924667835235596e-05,
-    "train_runtime": 5955.7987,
-    "train_samples": 160,
-    "train_samples_per_second": 0.054,
-    "train_steps_per_second": 0.003
+    "train_loss": 1.61745585501194e-05,
+    "train_runtime": 5416.7432,
+    "train_samples": 28,
+    "train_samples_per_second": 0.059,
+    "train_steps_per_second": 0.004
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 5.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 364.46875,
-      "epoch": 0.1,
-      "grad_norm": 10.924213409423828,
+      "completion_length": 521.65625,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 8.373077392578125,
       "kl": 0.0,
-      "learning_rate": 4.965903258506806e-07,
+      "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 0.689340058772359,
-      "reward_std": 0.6245058543572668,
-      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "reward": 0.30231481534428895,
+      "reward_std": 0.4286866073598503,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.31099630816606805,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.24297107267193496,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1830312479287386,
+      "rewards/xmlcount_reward_func": 0.05934374779462814,
       "step": 2
     },
     {
-      "completion_length": 320.9375,
-      "epoch": 0.2,
-      "grad_norm": 32.826290130615234,
-      "kl": 0.001088391010853229,
-      "learning_rate": 4.698684378016222e-07,
+      "completion_length": 365.7916666666667,
+      "epoch": 1.0,
+      "grad_norm": 3.361116409301758,
+      "kl": 0.005125638218790603,
+      "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.5512943307403475,
-      "reward_std": 0.44788288601557724,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.5099446430491904,
+      "reward_std": 0.4251110314701994,
+      "rewards/concensus_correctness_reward_func": 0.010416666666666666,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.22379432246088982,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.26819465138639015,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2024999945424497,
+      "rewards/xmlcount_reward_func": 0.14799999756117663,
       "step": 4
     },
     {
-      "completion_length": 350.3125,
-      "epoch": 0.3,
-      "grad_norm": 7.308592796325684,
-      "kl": 0.002020979256485589,
-      "learning_rate": 4.193203929064353e-07,
+      "completion_length": 441.21875,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 98.18972778320312,
+      "kl": 0.008048989489907399,
+      "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.6600665213773027,
-      "reward_std": 1.4531329879246186,
-      "rewards/concensus_correctness_reward_func": 0.7640000227838755,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.5239956639998127,
+      "reward_std": 0.5359824550687335,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.33031652320642024,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.25355815392686054,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2532500000670552,
+      "rewards/xmlcount_reward_func": 0.2079374985769391,
       "step": 6
     },
     {
-      "completion_length": 298.96875,
-      "epoch": 0.4,
-      "grad_norm": 82.434326171875,
-      "kl": 0.02994685058729374,
-      "learning_rate": 3.5042385616324236e-07,
+      "completion_length": 439.4166666666667,
+      "epoch": 2.0,
+      "grad_norm": 5.059068202972412,
+      "kl": 0.0012804252522376676,
+      "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.2903278581798077,
-      "reward_std": 1.3675250202722964,
-      "rewards/concensus_correctness_reward_func": 0.6431875005364418,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.6898338493580619,
+      "reward_std": 0.5673091932779547,
+      "rewards/concensus_correctness_reward_func": 0.010416666666666666,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3069841219112277,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.26758386722455424,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.21515625398023985,
+      "rewards/xmlcount_reward_func": 0.3284999914467335,
       "step": 8
     },
     {
-      "completion_length": 299.0625,
-      "epoch": 0.5,
-      "grad_norm": 5.223649024963379,
-      "kl": 0.3414705659815809,
-      "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0003,
-      "reward": 0.6657017794204876,
-      "reward_std": 0.4655185974552296,
+      "completion_length": 418.03125,
+      "epoch": 2.571428571428571,
+      "grad_norm": 8.908088684082031,
+      "kl": 0.12081245142326225,
+      "learning_rate": 3.1137137178519977e-07,
+      "loss": 0.0001,
+      "reward": 0.42404140171129256,
+      "reward_std": 0.33405227336334065,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.3487642778782174,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.2105101440101862,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1919374936260283,
+      "rewards/xmlcount_reward_func": 0.15103125385940075,
       "step": 10
     },
     {
-      "completion_length": 334.09375,
-      "epoch": 0.6,
-      "grad_norm": 5.859105587005615,
-      "kl": 0.003183810185873881,
-      "learning_rate": 1.886286282148002e-07,
+      "completion_length": 415.8333333333333,
+      "epoch": 3.0,
+      "grad_norm": 3.142291784286499,
+      "kl": 0.0036896999566427744,
+      "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.6069466369226575,
-      "reward_std": 0.5879159582545981,
-      "rewards/concensus_correctness_reward_func": 0.031687501817941666,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.5049940745035807,
+      "reward_std": 0.711844447379311,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3816966563463211,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.399119071662426,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06856249796692282,
+      "rewards/xmlcount_reward_func": 0.02254166081547737,
       "step": 12
     },
     {
-      "completion_length": 333.40625,
-      "epoch": 0.7,
-      "grad_norm": 4.700389385223389,
-      "kl": 0.004205428318527993,
-      "learning_rate": 1.1326296046939333e-07,
+      "completion_length": 386.625,
+      "epoch": 3.571428571428571,
+      "grad_norm": 31.762287139892578,
+      "kl": 0.01464658961049281,
+      "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.3486387468874454,
-      "reward_std": 1.5692350980825722,
-      "rewards/concensus_correctness_reward_func": 0.6490624994039536,
+      "reward": 0.5932526526739821,
+      "reward_std": 0.7520454289624467,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3849825127981603,
+      "rewards/question_recreation_reward_func": 0.3564401565818116,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.25209374679252505,
+      "rewards/xmlcount_reward_func": 0.16649999655783176,
       "step": 14
     },
     {
-      "completion_length": 381.28125,
-      "epoch": 0.8,
-      "grad_norm": 5.702529430389404,
-      "kl": 0.002874413097742945,
-      "learning_rate": 5.271487265090163e-08,
+      "completion_length": 567.5833333333334,
+      "epoch": 4.0,
+      "grad_norm": 2.1845624446868896,
+      "kl": 0.0059801468572307686,
+      "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.5994385657832026,
-      "reward_std": 0.8584566740319133,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.37394068999371183,
+      "reward_std": 0.6022146987573554,
+      "rewards/concensus_correctness_reward_func": 0.010500000168879827,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3692510575056076,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.16994070855434984,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16768749989569187,
+      "rewards/xmlcount_reward_func": 0.19349999477465948,
       "step": 16
     },
     {
-      "completion_length": 312.875,
-      "epoch": 0.9,
-      "grad_norm": 6.588188648223877,
-      "kl": 0.004472060274565592,
-      "learning_rate": 1.3545689574841341e-08,
+      "completion_length": 341.21875,
+      "epoch": 4.571428571428571,
+      "grad_norm": 15.037383079528809,
+      "kl": 0.0031782291844137944,
+      "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.6218310541007668,
-      "reward_std": 0.5228257608832791,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 1.0048452126793563,
+      "reward_std": 1.3991620691376738,
+      "rewards/concensus_correctness_reward_func": 0.625,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.2912998090032488,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.29072020755847916,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2055312478914857,
+      "rewards/xmlcount_reward_func": 0.026624999940395355,
       "step": 18
     },
     {
-      "completion_length": 341.375,
-      "epoch": 1.0,
-      "grad_norm": 6.06608772277832,
-      "kl": 0.0028928506580996327,
-      "learning_rate": 0.0,
+      "completion_length": 481.7916666666667,
+      "epoch": 5.0,
+      "grad_norm": 3.4167587757110596,
+      "kl": 0.0033248904801439494,
+      "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.3756030010990798,
-      "reward_std": 1.148384967003949,
-      "rewards/concensus_correctness_reward_func": 0.640625,
+      "reward": 0.14136970741674304,
+      "reward_std": 0.6498097935691476,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.32694675703532994,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.23311969513694444,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.15803125128149986,
+      "rewards/xmlcount_reward_func": -0.09174999222159386,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.924667835235596e-05,
-      "train_runtime": 5955.7987,
-      "train_samples_per_second": 0.054,
-      "train_steps_per_second": 0.003
+      "train_loss": 1.61745585501194e-05,
+      "train_runtime": 5416.7432,
+      "train_samples_per_second": 0.059,
+      "train_steps_per_second": 0.004
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 7,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a00c4021a51dcdc9997be179384445924a71ab366952f9ee7a4256349f99ded8
-size 6008
+oid sha256:d122afff409e31df56d1daf929b452aadea37245c08408ec42ecb17a98d9436a
+size 5944