End of training

README.md

@@ -38,7 +38,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.50.3
+- Transformers: 4.51.3
 - Pytorch: 2.5.1
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.527531564235687e-06,
-    "train_runtime": 2404.5061,
-    "train_samples": 160,
-    "train_samples_per_second": 0.133,
-    "train_steps_per_second": 0.008
+    "train_loss": 0.0002281336463511252,
+    "train_runtime": 4153.4805,
+    "train_samples": 28,
+    "train_samples_per_second": 0.077,
+    "train_steps_per_second": 0.005
 }

config.json

@@ -21,7 +21,7 @@
   "sliding_window": 32768,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.50.3",
+  "transformers_version": "4.51.3",
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936

generation_config.json

@@ -10,5 +10,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.50.3"
+  "transformers_version": "4.51.3"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70243a9a768ca6c72a376d2a9989a37120a13cd018a515f1b9578b56fefd5712
+oid sha256:a76ea720810675106aa609af81fd8eab3b1b9fc5402fd06278ec336186ffe1ec
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.527531564235687e-06,
-    "train_runtime": 2404.5061,
-    "train_samples": 160,
-    "train_samples_per_second": 0.133,
-    "train_steps_per_second": 0.008
+    "train_loss": 0.0002281336463511252,
+    "train_runtime": 4153.4805,
+    "train_samples": 28,
+    "train_samples_per_second": 0.077,
+    "train_steps_per_second": 0.005
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 5.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 316.71875,
-      "epoch": 0.1,
-      "grad_norm": 20.640230178833008,
+      "completion_length": 308.15625,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 41.633419036865234,
       "kl": 0.0,
-      "learning_rate": 4.965903258506806e-07,
-      "loss": -0.0,
-      "reward": 1.0204834319010843,
-      "reward_std": 1.2534251582983416,
-      "rewards/concensus_correctness_reward_func": 0.625,
-      "rewards/consensus_reward_func": 0.0,
+      "learning_rate": 5e-07,
+      "loss": 0.0,
+      "reward": 0.40645824919920415,
+      "reward_std": 0.31749973719843183,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.25579588310210966,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2209270077291876,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07718750182539225,
+      "rewards/xmlcount_reward_func": 0.12303124996833503,
       "step": 2
     },
     {
-      "completion_length": 333.21875,
-      "epoch": 0.2,
-      "grad_norm": 5.591650009155273,
-      "kl": 0.0014399742321984377,
-      "learning_rate": 4.698684378016222e-07,
+      "completion_length": 365.7083333333333,
+      "epoch": 1.0,
+      "grad_norm": 3.1837849617004395,
+      "kl": 0.0010262345652639244,
+      "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.6284063304774463,
-      "reward_std": 0.6146276032086462,
+      "reward": 0.6313755170752605,
+      "reward_std": 0.5310685941561436,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.294125072658062,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.2688755051543315,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.20928124897181988,
+      "rewards/xmlcount_reward_func": 0.19583333159486452,
       "step": 4
     },
     {
-      "completion_length": 337.15625,
-      "epoch": 0.3,
-      "grad_norm": 8.498673439025879,
-      "kl": 0.002181062922318233,
-      "learning_rate": 4.193203929064353e-07,
+      "completion_length": 365.15625,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 7.127258777618408,
+      "kl": 0.0020578340663632844,
+      "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.8236608348088339,
-      "reward_std": 0.40198390794103034,
-      "rewards/concensus_correctness_reward_func": 0.06043750233948231,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.7601827555336058,
+      "reward_std": 0.3503397152235266,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3301920867525041,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.34671399975195527,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3705312458187109,
+      "rewards/xmlcount_reward_func": 0.2259687555488199,
       "step": 6
     },
     {
-      "completion_length": 315.34375,
-      "epoch": 0.4,
-      "grad_norm": 9.28067398071289,
-      "kl": 0.0012329019318713108,
-      "learning_rate": 3.5042385616324236e-07,
+      "completion_length": 314.9583333333333,
+      "epoch": 2.0,
+      "grad_norm": 2.7644894123077393,
+      "kl": 0.04608077997302947,
+      "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.9061323929345235,
-      "reward_std": 1.2012234527501278,
-      "rewards/concensus_correctness_reward_func": 0.625,
+      "reward": 0.4097979703607659,
+      "reward_std": 0.3929105601273477,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.14297609927598387,
+      "rewards/question_recreation_reward_func": 0.25896464427933097,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13815624825656414,
+      "rewards/xmlcount_reward_func": 0.15083333291113377,
       "step": 8
     },
     {
-      "completion_length": 396.6875,
-      "epoch": 0.5,
-      "grad_norm": 6.3696184158325195,
-      "kl": 0.006439117893023649,
-      "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0,
-      "reward": 0.9482769472524524,
-      "reward_std": 0.6139591162791476,
-      "rewards/concensus_correctness_reward_func": 0.05256250128149986,
+      "completion_length": 344.625,
+      "epoch": 2.571428571428571,
+      "grad_norm": 58.070220947265625,
+      "kl": 0.3261278549889539,
+      "learning_rate": 3.1137137178519977e-07,
+      "loss": 0.0003,
+      "reward": 0.424428222788265,
+      "reward_std": 0.4320046803622972,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3935582006815821,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.205803226766875,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.439656249480322,
+      "rewards/xmlcount_reward_func": 0.21862499974668026,
       "step": 10
     },
     {
-      "completion_length": 317.9375,
-      "epoch": 0.6,
-      "grad_norm": 8.388046264648438,
-      "kl": 0.006362274449202232,
-      "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0,
-      "reward": 0.6173857575049624,
-      "reward_std": 0.45685271653928794,
-      "rewards/concensus_correctness_reward_func": 0.018437499180436134,
-      "rewards/consensus_reward_func": 0.0625,
+      "completion_length": 270.1666666666667,
+      "epoch": 3.0,
+      "grad_norm": 109.35530853271484,
+      "kl": 0.5230223730419917,
+      "learning_rate": 2.2935516363191693e-07,
+      "loss": 0.0004,
+      "reward": 1.480574295700838,
+      "reward_std": 1.5046859136006485,
+      "rewards/concensus_correctness_reward_func": 0.8659999991456667,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.30632324831094593,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.2699492978863418,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16762500000186265,
+      "rewards/xmlcount_reward_func": 0.17795833324392638,
       "step": 12
     },
     {
-      "completion_length": 328.0625,
-      "epoch": 0.7,
-      "grad_norm": 6.688593864440918,
-      "kl": 0.003958331943067606,
-      "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0,
-      "reward": 0.6333054774440825,
-      "reward_std": 0.5454679431859404,
+      "completion_length": 303.90625,
+      "epoch": 3.571428571428571,
+      "grad_norm": 104.08259582519531,
+      "kl": 1.4068435160588706,
+      "learning_rate": 1.4957614383675767e-07,
+      "loss": 0.0014,
+      "reward": 0.48999457189347595,
+      "reward_std": 0.4255776608479209,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.3044929875759408,
+      "rewards/question_recreation_reward_func": 0.19811958575155586,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.14131249953061342,
+      "rewards/xmlcount_reward_func": 0.16687499592080712,
       "step": 14
     },
     {
-      "completion_length": 309.5625,
-      "epoch": 0.8,
-      "grad_norm": 12.809540748596191,
-      "kl": 0.011681057876558043,
-      "learning_rate": 5.271487265090163e-08,
+      "completion_length": 325.375,
+      "epoch": 4.0,
+      "grad_norm": 8.133492469787598,
+      "kl": 0.006218782771611586,
+      "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.933492950105574,
-      "reward_std": 0.33618732055765577,
-      "rewards/concensus_correctness_reward_func": 1.25,
+      "reward": 0.5689306681354841,
+      "reward_std": 0.5994748280694088,
+      "rewards/concensus_correctness_reward_func": 0.010416666666666666,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.32083665631944314,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.3621806618757546,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.23765625635860488,
+      "rewards/xmlcount_reward_func": 0.11299999923600505,
       "step": 16
     },
     {
-      "completion_length": 316.84375,
-      "epoch": 0.9,
-      "grad_norm": 4.517097473144531,
-      "kl": 0.008473690730170347,
-      "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0,
-      "reward": 0.4718033701647073,
-      "reward_std": 0.536688668653369,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "completion_length": 267.6875,
+      "epoch": 4.571428571428571,
+      "grad_norm": 99.26972961425781,
+      "kl": 0.10651466252238606,
+      "learning_rate": 3.013156219837776e-08,
+      "loss": 0.0001,
+      "reward": 0.4112930775154382,
+      "reward_std": 0.2777931771270232,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.23252211860381067,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.16966809332370758,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17678125109523535,
+      "rewards/xmlcount_reward_func": 0.10881249979138374,
       "step": 18
     },
     {
-      "completion_length": 279.1875,
-      "epoch": 1.0,
-      "grad_norm": 61.814781188964844,
-      "kl": 0.003459241639575339,
-      "learning_rate": 0.0,
+      "completion_length": 293.25,
+      "epoch": 5.0,
+      "grad_norm": 22.004924774169922,
+      "kl": 0.0053467367445894825,
+      "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.3219008122105151,
-      "reward_std": 1.3056973450875375,
-      "rewards/concensus_correctness_reward_func": 0.625,
+      "reward": 0.5409725056185076,
+      "reward_std": 0.746750327021194,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.27558836503885686,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.28509751621944207,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1713124974630773,
+      "rewards/xmlcount_reward_func": 0.0892083328217268,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 4.527531564235687e-06,
-      "train_runtime": 2404.5061,
-      "train_samples_per_second": 0.133,
-      "train_steps_per_second": 0.008
+      "train_loss": 0.0002281336463511252,
+      "train_runtime": 4153.4805,
+      "train_samples_per_second": 0.077,
+      "train_steps_per_second": 0.005
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 7,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19c7b2f71fe47069001777accee89e0e0486f2a474360539fa941b7c1af496b4
-size 6008
+oid sha256:f641f78431443ce2a0e481d8fbfceff10f3651aad7b19eaade250a6be37d949a
+size 5944