End of training

README.md

@@ -39,7 +39,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.51.3
+- Transformers: 4.48.2
 - Pytorch: 2.5.1
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1

adapter_config.json

@@ -25,12 +25,12 @@
   "revision": null,
   "target_modules": [
     "down_proj",
-    "o_proj",
+    "gate_proj",
     "up_proj",
     "q_proj",
-    "k_proj",
+    "o_proj",
     "v_proj",
-    "gate_proj"
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c8a2df1b5d0c452b4359705d2f4ad6ed177a863983a54c11dd2e48c60eb8426
+oid sha256:21a0a3d9986c4ffa2c89e9c360615aa17fd1414ac127d27c066dcf72db059e39
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.8439161976857577e-05,
-    "train_runtime": 391.7395,
-    "train_samples": 11,
-    "train_samples_per_second": 0.817,
-    "train_steps_per_second": 0.051
+    "train_loss": 8.563369419789524e-07,
+    "train_runtime": 1151.3592,
+    "train_samples": 19,
+    "train_samples_per_second": 0.278,
+    "train_steps_per_second": 0.017
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.8439161976857577e-05,
-    "train_runtime": 391.7395,
-    "train_samples": 11,
-    "train_samples_per_second": 0.817,
-    "train_steps_per_second": 0.051
+    "train_loss": 8.563369419789524e-07,
+    "train_runtime": 1151.3592,
+    "train_samples": 19,
+    "train_samples_per_second": 0.278,
+    "train_steps_per_second": 0.017
 }

trainer_state.json

@@ -1,8 +1,7 @@
 {
-  "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 6.7272727272727275,
+  "epoch": 4.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +9,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 315.4375,
-      "epoch": 0.7272727272727273,
-      "grad_norm": 0.43634098768234253,
-      "kl": 0.01944660220760852,
-      "learning_rate": 5e-07,
+      "completion_length": 411.46875,
+      "epoch": 0.42105263157894735,
+      "grad_norm": 0.37723931670188904,
+      "kl": 0.0007731288496870548,
+      "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 1.9734177440404892,
-      "reward_std": 0.8466662056744099,
-      "rewards/concensus_correctness_reward_func": 0.19637500029057264,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 1.3933892026543617,
+      "reward_std": 1.315464586019516,
+      "rewards/concensus_correctness_reward_func": 0.015687499195337296,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6958552822470665,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.44254542514681816,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.44056250154972076,
+      "rewards/xmlcount_reward_func": 0.23203124664723873,
       "step": 2
     },
     {
-      "completion_length": 305.60714285714283,
-      "epoch": 1.3636363636363638,
-      "grad_norm": 0.5860344767570496,
-      "kl": 0.019787386591945375,
-      "learning_rate": 4.864543104251586e-07,
+      "completion_length": 427.28125,
+      "epoch": 0.8421052631578947,
+      "grad_norm": 0.38159775733947754,
+      "kl": 0.0016558529023313895,
+      "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 1.811240485736302,
-      "reward_std": 1.033715465239116,
-      "rewards/concensus_correctness_reward_func": 0.029428571462631226,
-      "rewards/consensus_reward_func": 0.5714285714285714,
+      "reward": 2.5967174619436264,
+      "reward_std": 1.4946671575307846,
+      "rewards/concensus_correctness_reward_func": 0.18375000171363354,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5268833254064832,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.6685924418270588,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.6656428607446807,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.43187500536441803,
       "step": 4
     },
     {
-      "completion_length": 379.64285714285717,
-      "epoch": 2.0,
-      "grad_norm": 0.3656754791736603,
-      "kl": 0.017877236141690185,
-      "learning_rate": 4.472851273490984e-07,
+      "completion_length": 605.0357142857143,
+      "epoch": 1.2105263157894737,
+      "grad_norm": 0.32786497473716736,
+      "kl": 0.0006202836063623961,
+      "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 2.165443650313786,
-      "reward_std": 1.0742863927568709,
-      "rewards/concensus_correctness_reward_func": 0.2620000006364925,
-      "rewards/consensus_reward_func": 0.6428571428571429,
+      "reward": 1.8956254294940404,
+      "reward_std": 1.709733017853328,
+      "rewards/concensus_correctness_reward_func": 0.1756428544010435,
+      "rewards/consensus_reward_func": 0.42857142857142855,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7569794399397713,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.4678928405046463,
+      "rewards/final_correctness_reward_func": 0.6428571428571429,
+      "rewards/question_recreation_reward_func": 0.5850897569741521,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.0634642903293882,
       "step": 6
     },
     {
-      "completion_length": 342.15625,
-      "epoch": 2.7272727272727275,
-      "grad_norm": 0.7130538821220398,
-      "kl": 0.019564266316592693,
-      "learning_rate": 3.867370395306068e-07,
+      "completion_length": 396.9375,
+      "epoch": 1.631578947368421,
+      "grad_norm": 0.693664014339447,
+      "kl": 0.0008750769047765061,
+      "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 1.9031648635864258,
-      "reward_std": 1.2054845029488206,
-      "rewards/concensus_correctness_reward_func": 0.10531250271014869,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 1.735365480184555,
+      "reward_std": 0.8610958196222782,
+      "rewards/concensus_correctness_reward_func": 0.06437500100582838,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6045398563146591,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.4995216941460967,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.5839375080540776,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.46834377385675907,
       "step": 8
     },
     {
-      "completion_length": 332.32142857142856,
-      "epoch": 3.3636363636363638,
-      "grad_norm": 0.6676836013793945,
-      "kl": 0.018274167286498205,
-      "learning_rate": 3.1137137178519977e-07,
+      "completion_length": 423.82142857142856,
+      "epoch": 2.0,
+      "grad_norm": 0.44714874029159546,
+      "kl": 0.0006811664206907153,
+      "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 1.851676276751927,
-      "reward_std": 1.3029427783829826,
-      "rewards/concensus_correctness_reward_func": 0.08992857166699,
-      "rewards/consensus_reward_func": 0.7142857142857143,
+      "reward": 1.9886690718787057,
+      "reward_std": 1.4905504882335663,
+      "rewards/concensus_correctness_reward_func": 0.1527857163122722,
+      "rewards/consensus_reward_func": 0.5714285714285714,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5192120053938457,
+      "rewards/final_correctness_reward_func": 0.35714285714285715,
+      "rewards/question_recreation_reward_func": 0.4890619324786322,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03571428571428571,
-      "rewards/xmlcount_reward_func": 0.4925357241715704,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.40039284633738653,
       "step": 10
     },
     {
-      "completion_length": 353.14285714285717,
-      "epoch": 4.0,
-      "grad_norm": 0.4082489013671875,
-      "kl": 0.023155115943934237,
-      "learning_rate": 2.2935516363191693e-07,
+      "completion_length": 421.125,
+      "epoch": 2.4210526315789473,
+      "grad_norm": 0.43313491344451904,
+      "kl": 0.0007299965582205914,
+      "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 1.6882215908595495,
-      "reward_std": 1.1843782918793815,
-      "rewards/concensus_correctness_reward_func": 0.1665714287332126,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 1.7385327890515327,
+      "reward_std": 1.2362245097756386,
+      "rewards/concensus_correctness_reward_func": 0.015687499195337296,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.661828705242702,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.49909529834985733,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.43125001021793913,
+      "rewards/xmlcount_reward_func": 0.34875000827014446,
       "step": 12
     },
     {
-      "completion_length": 330.0,
-      "epoch": 4.7272727272727275,
-      "grad_norm": 0.6438230872154236,
-      "kl": 0.017973024281673133,
-      "learning_rate": 1.4957614383675767e-07,
+      "completion_length": 420.34375,
+      "epoch": 2.8421052631578947,
+      "grad_norm": 0.3356363773345947,
+      "kl": 0.0006936805220902897,
+      "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 1.814220629632473,
-      "reward_std": 1.0252906922250986,
-      "rewards/concensus_correctness_reward_func": 0.113625000230968,
+      "reward": 1.9652772545814514,
+      "reward_std": 1.3554531522095203,
+      "rewards/concensus_correctness_reward_func": 0.28974999859929085,
       "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5479393489658833,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.4195272733923048,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.6370312441140413,
+      "rewards/xmlcount_reward_func": 0.3028750028461218,
       "step": 14
     },
     {
-      "completion_length": 380.7142857142857,
-      "epoch": 5.363636363636363,
-      "grad_norm": 0.5655611157417297,
-      "kl": 0.02232003424848829,
-      "learning_rate": 8.067960709356478e-08,
+      "completion_length": 568.6071428571429,
+      "epoch": 3.2105263157894735,
+      "grad_norm": 0.5204138159751892,
+      "kl": 0.0012830742801140463,
+      "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 1.9248718449047633,
-      "reward_std": 0.9834700056484768,
-      "rewards/concensus_correctness_reward_func": 0.1685714317219598,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.8036100949559892,
+      "reward_std": 1.5157660841941833,
+      "rewards/concensus_correctness_reward_func": 0.19057142521653855,
+      "rewards/consensus_reward_func": 0.6428571428571429,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7034433058329991,
+      "rewards/final_correctness_reward_func": 0.35714285714285715,
+      "rewards/question_recreation_reward_func": 0.52464581068073,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.5349999921662467,
+      "rewards/xmlcount_reward_func": 0.07053571726594653,
       "step": 16
     },
     {
-      "completion_length": 337.64285714285717,
-      "epoch": 6.0,
-      "grad_norm": 0.5070009827613831,
-      "kl": 0.01860570947506598,
-      "learning_rate": 3.013156219837776e-08,
+      "completion_length": 342.96875,
+      "epoch": 3.6315789473684212,
+      "grad_norm": 0.8073804378509521,
+      "kl": 0.0008730229856155347,
+      "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 1.819380453654698,
-      "reward_std": 1.2818671422345298,
-      "rewards/concensus_correctness_reward_func": 0.05571428579943521,
-      "rewards/consensus_reward_func": 0.6428571428571429,
+      "reward": 1.6936715841293335,
+      "reward_std": 0.9070026399567723,
+      "rewards/concensus_correctness_reward_func": 0.04506250098347664,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.634559052331107,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.608390349894762,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.3969642903123583,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.5089687407016754,
       "step": 18
     },
     {
-      "completion_length": 354.09375,
-      "epoch": 6.7272727272727275,
-      "grad_norm": 0.5374490022659302,
-      "kl": 0.02240668353624642,
-      "learning_rate": 3.4096741493194193e-09,
+      "completion_length": 432.32142857142856,
+      "epoch": 4.0,
+      "grad_norm": 0.32386574149131775,
+      "kl": 0.0007003966311458498,
+      "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 2.309537395834923,
-      "reward_std": 1.2209416665136814,
-      "rewards/concensus_correctness_reward_func": 0.08237500069662929,
-      "rewards/consensus_reward_func": 0.875,
+      "reward": 1.721566915512085,
+      "reward_std": 1.5520981379917689,
+      "rewards/concensus_correctness_reward_func": 0.04321428494794028,
+      "rewards/consensus_reward_func": 0.6428571428571429,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5994749031960964,
+      "rewards/final_correctness_reward_func": 0.42857142857142855,
+      "rewards/question_recreation_reward_func": 0.45238833342279705,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.6901874914765358,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.15453571719782694,
       "step": 20
     },
     {
-      "epoch": 6.7272727272727275,
+      "epoch": 4.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.8439161976857577e-05,
-      "train_runtime": 391.7395,
-      "train_samples_per_second": 0.817,
-      "train_steps_per_second": 0.051
+      "train_loss": 8.563369419789524e-07,
+      "train_runtime": 1151.3592,
+      "train_samples_per_second": 0.278,
+      "train_steps_per_second": 0.017
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 10,
+  "num_train_epochs": 5,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e0f2b5783e918c26f6a28882af8661739238dd1128ee78a43929209a2e2c34a
-size 6008
+oid sha256:baab2a73beabadd709f675acd0fe11871d46a58145cc5934ac293068f8816cf2
+size 5944