End of training

README.md

@@ -39,9 +39,9 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.51.3
-- Pytorch: 2.6.0
-- Datasets: 3.5.1
+- Transformers: 4.48.2
+- Pytorch: 2.5.1
+- Datasets: 3.6.0
 - Tokenizers: 0.21.1
 
 ## Citations

adapter_config.json

@@ -25,12 +25,12 @@
   "revision": null,
   "target_modules": [
     "o_proj",
-    "q_proj",
-    "down_proj",
+    "v_proj",
     "k_proj",
-    "up_proj",
+    "down_proj",
     "gate_proj",
-    "v_proj"
+    "up_proj",
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fac438b05191a7bda1fab3394196169b11ed24e50900562a2bac9d380a90f73d
+oid sha256:e8d5cd32c238d5128f1669c54131ccdd2d7efb5b7dd3a408098c20df45165f7e
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.005897418595850468,
-    "train_runtime": 233.2129,
+    "train_loss": 7.507646330395801e-07,
+    "train_runtime": 324.7295,
     "train_samples": 14,
-    "train_samples_per_second": 1.372,
-    "train_steps_per_second": 0.086
+    "train_samples_per_second": 0.985,
+    "train_steps_per_second": 0.062
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.005897418595850468,
-    "train_runtime": 233.2129,
+    "train_loss": 7.507646330395801e-07,
+    "train_runtime": 324.7295,
     "train_samples": 14,
-    "train_samples_per_second": 1.372,
-    "train_steps_per_second": 0.086
+    "train_samples_per_second": 0.985,
+    "train_steps_per_second": 0.062
 }

trainer_state.json

@@ -1,5 +1,4 @@
 {
-  "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 5.0,
@@ -10,203 +9,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 205.9375,
+      "completion_length": 307.84375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 2.4009063243865967,
-      "kl": 2.8026754818856716,
-      "learning_rate": 5e-07,
-      "loss": 0.0028,
-      "reward": 12.936573654413223,
-      "reward_std": 8.837230630218983,
-      "rewards/concensus_correctness_reward_func": 9.153625011444092,
-      "rewards/consensus_reward_func": 1.0625,
+      "grad_norm": 0.47697943449020386,
+      "kl": 0.0007509570168622304,
+      "learning_rate": 4.965903258506806e-07,
+      "loss": 0.0,
+      "reward": 1.2545952871441841,
+      "reward_std": 1.0445544943213463,
+      "rewards/concensus_correctness_reward_func": 0.04481250047683716,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0625,
-      "rewards/question_recreation_reward_func": 0.39913596771657467,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.024437502026558,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4747202843427658,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.34443750232458115,
       "step": 2
     },
     {
-      "completion_length": 193.95833333333334,
+      "completion_length": 283.7916666666667,
       "epoch": 1.0,
-      "grad_norm": 1.7550156116485596,
-      "kl": 4.095069532593091,
-      "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0031,
-      "reward": 14.291598002115885,
-      "reward_std": 11.781408150990805,
-      "rewards/concensus_correctness_reward_func": 10.136749982833862,
-      "rewards/consensus_reward_func": 1.0,
+      "grad_norm": 0.4536288380622864,
+      "kl": 0.000830039381980896,
+      "learning_rate": 4.698684378016222e-07,
+      "loss": 0.0,
+      "reward": 3.9119001924991608,
+      "reward_std": 4.873037169377009,
+      "rewards/concensus_correctness_reward_func": 2.0474166671435037,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.4733063131570816,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.46273347238699597,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.0148750046888988,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.235083336631457,
       "step": 4
     },
     {
-      "completion_length": 211.875,
+      "completion_length": 311.71875,
       "epoch": 1.5714285714285714,
-      "grad_norm": 12.002602577209473,
-      "kl": 11.966277405619621,
-      "learning_rate": 4.472851273490984e-07,
-      "loss": 0.012,
-      "reward": 15.091071665287018,
-      "reward_std": 10.066041484475136,
-      "rewards/concensus_correctness_reward_func": 10.816874980926514,
-      "rewards/consensus_reward_func": 1.0625,
+      "grad_norm": 0.68195641040802,
+      "kl": 0.0008154660972650163,
+      "learning_rate": 4.193203929064353e-07,
+      "loss": 0.0,
+      "reward": 2.5976146487519145,
+      "reward_std": 2.239759039133787,
+      "rewards/concensus_correctness_reward_func": 1.302437499165535,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.5257593058049679,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.37242713011801243,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.0609374940395355,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.36024999991059303,
       "step": 6
     },
     {
-      "completion_length": 186.58333333333334,
+      "completion_length": 286.9166666666667,
       "epoch": 2.0,
-      "grad_norm": 1.675034999847412,
-      "kl": 3.8484922597805657,
-      "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0029,
-      "reward": 14.593044877052307,
-      "reward_std": 8.397790829340616,
-      "rewards/concensus_correctness_reward_func": 10.0,
-      "rewards/consensus_reward_func": 1.25,
+      "grad_norm": 0.3821156620979309,
+      "kl": 0.0008516140417971959,
+      "learning_rate": 3.5042385616324236e-07,
+      "loss": 0.0,
+      "reward": 4.207462310791016,
+      "reward_std": 2.887537010014057,
+      "rewards/concensus_correctness_reward_func": 1.8768334239721298,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0833333333333333,
-      "rewards/question_recreation_reward_func": 0.6655867447455724,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.5367539475361506,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.2191249926884968,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3772083173195521,
       "step": 8
     },
     {
-      "completion_length": 200.3125,
+      "completion_length": 296.90625,
       "epoch": 2.571428571428571,
-      "grad_norm": 5.33963680267334,
-      "kl": 2.191370403394103,
-      "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0022,
-      "reward": 14.482143640518188,
-      "reward_std": 9.357158794999123,
-      "rewards/concensus_correctness_reward_func": 10.73700000345707,
-      "rewards/consensus_reward_func": 1.0625,
+      "grad_norm": 0.5019404888153076,
+      "kl": 0.0008093566793831997,
+      "learning_rate": 2.706448363680831e-07,
+      "loss": 0.0,
+      "reward": 2.2707934752106667,
+      "reward_std": 2.4034877978265285,
+      "rewards/concensus_correctness_reward_func": 0.7142499992623925,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.875,
-      "rewards/question_recreation_reward_func": 0.47701864689588547,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.43176219426095486,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.0337500106543303,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.31228123791515827,
       "step": 10
     },
     {
-      "completion_length": 199.04166666666666,
+      "completion_length": 300.4166666666667,
       "epoch": 3.0,
-      "grad_norm": 56.0693244934082,
-      "kl": 23.470269282658894,
-      "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0176,
-      "reward": 13.0904301404953,
-      "reward_std": 9.864758412043253,
-      "rewards/concensus_correctness_reward_func": 8.748999834060669,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "grad_norm": 0.35668668150901794,
+      "kl": 0.0008771180194647362,
+      "learning_rate": 1.886286282148002e-07,
+      "loss": 0.0,
+      "reward": 1.604244440793991,
+      "reward_std": 1.1329847822586696,
+      "rewards/concensus_correctness_reward_func": 0.4125833222642541,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1666666666666667,
-      "rewards/question_recreation_reward_func": 0.5238051414489746,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.462744377553463,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3333333333333333,
-      "rewards/xmlcount_reward_func": 1.1509583592414856,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.145583334689339,
       "step": 12
     },
     {
-      "completion_length": 175.1875,
+      "completion_length": 339.65625,
       "epoch": 3.571428571428571,
-      "grad_norm": 2.404494285583496,
-      "kl": 6.291187301278114,
-      "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0063,
-      "reward": 16.365968823432922,
-      "reward_std": 8.914833560585976,
-      "rewards/concensus_correctness_reward_func": 12.097187519073486,
-      "rewards/consensus_reward_func": 1.1875,
+      "grad_norm": 0.48297348618507385,
+      "kl": 0.0007411040933220647,
+      "learning_rate": 1.1326296046939333e-07,
+      "loss": 0.0,
+      "reward": 3.51069688051939,
+      "reward_std": 3.406370159238577,
+      "rewards/concensus_correctness_reward_func": 1.5621250122785568,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.125,
-      "rewards/question_recreation_reward_func": 0.5064689349383116,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.664228143170476,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.1060625240206718,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.2530937436968088,
       "step": 14
     },
     {
-      "completion_length": 189.66666666666666,
+      "completion_length": 309.3333333333333,
       "epoch": 4.0,
-      "grad_norm": 3.134777307510376,
-      "kl": 7.780150505403678,
-      "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0058,
-      "reward": 13.454876979192099,
-      "reward_std": 7.809763252735138,
-      "rewards/concensus_correctness_reward_func": 9.576916615168253,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "grad_norm": 0.23572970926761627,
+      "kl": 0.0007775598302638779,
+      "learning_rate": 5.271487265090163e-08,
+      "loss": 0.0,
+      "reward": 1.7949834987521172,
+      "reward_std": 2.426940143108368,
+      "rewards/concensus_correctness_reward_func": 0.8333333333333334,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9166666666666666,
-      "rewards/question_recreation_reward_func": 0.42221041520436603,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.3045668974518776,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.0599166850248973,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3237500103811423,
       "step": 16
     },
     {
-      "completion_length": 226.9375,
+      "completion_length": 297.15625,
       "epoch": 4.571428571428571,
-      "grad_norm": 3.7106289863586426,
-      "kl": 4.958254403434694,
-      "learning_rate": 3.013156219837776e-08,
-      "loss": 0.005,
-      "reward": 15.046832501888275,
-      "reward_std": 9.295212596654892,
-      "rewards/concensus_correctness_reward_func": 10.731562495231628,
-      "rewards/consensus_reward_func": 1.125,
+      "grad_norm": 0.7020504474639893,
+      "kl": 0.0008789005078142509,
+      "learning_rate": 1.3545689574841341e-08,
+      "loss": 0.0,
+      "reward": 2.893677331507206,
+      "reward_std": 2.640741240233183,
+      "rewards/concensus_correctness_reward_func": 1.3593749990686774,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.6867385134100914,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5348647553473711,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 0.9722812473773956,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.37443749979138374,
       "step": 18
     },
     {
-      "completion_length": 191.29166666666666,
+      "completion_length": 261.875,
       "epoch": 5.0,
-      "grad_norm": 2.2372097969055176,
-      "kl": 1.825108637412389,
-      "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0014,
-      "reward": 18.844095865885418,
-      "reward_std": 6.021364857753118,
-      "rewards/concensus_correctness_reward_func": 14.303416649500528,
-      "rewards/consensus_reward_func": 1.4166666666666667,
+      "grad_norm": 0.7190520763397217,
+      "kl": 0.001175893092295155,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "reward": 1.0453936755657196,
+      "reward_std": 0.862831545372804,
+      "rewards/concensus_correctness_reward_func": 0.09633333484331767,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.42892923206090927,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3342686705291271,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.0700833300749462,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.2606250022848447,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.005897418595850468,
-      "train_runtime": 233.2129,
-      "train_samples_per_second": 1.372,
-      "train_steps_per_second": 0.086
+      "train_loss": 7.507646330395801e-07,
+      "train_runtime": 324.7295,
+      "train_samples_per_second": 0.985,
+      "train_steps_per_second": 0.062
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:634058d47e7f99f4ab0c743b2515824e0e8607052e9d5c80746cc2c073c51727
-size 6008
+oid sha256:d44075cf81c5f290f02f1cc48c8eeebf93baed52612215d4ac69fb77dd1ee48d
+size 5944