End of training

README.md

@@ -39,7 +39,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.51.3
+- Transformers: 4.48.2
 - Pytorch: 2.5.1
 - Datasets: 3.6.0
 - Tokenizers: 0.21.1

adapter_config.json

@@ -24,13 +24,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "o_proj",
+    "up_proj",
+    "q_proj",
     "down_proj",
     "v_proj",
     "k_proj",
-    "gate_proj",
-    "q_proj",
-    "o_proj",
-    "up_proj"
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a5fa4a2dc1b3e99902918e584c0401423d58b40617c8503987d0ae3686f6716
+oid sha256:1a0154890f83a8a78d37ae3c21c27a51b097eb81aefa2791b9dee277c270c5b2
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0013065586477750911,
-    "train_runtime": 248.9001,
-    "train_samples": 14,
-    "train_samples_per_second": 1.286,
-    "train_steps_per_second": 0.08
+    "train_loss": 4.052198395099183e-06,
+    "train_runtime": 837.0301,
+    "train_samples": 63,
+    "train_samples_per_second": 0.382,
+    "train_steps_per_second": 0.024
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0013065586477750911,
-    "train_runtime": 248.9001,
-    "train_samples": 14,
-    "train_samples_per_second": 1.286,
-    "train_steps_per_second": 0.08
+    "train_loss": 4.052198395099183e-06,
+    "train_runtime": 837.0301,
+    "train_samples": 63,
+    "train_samples_per_second": 0.382,
+    "train_steps_per_second": 0.024
 }

trainer_state.json

@@ -1,8 +1,7 @@
 {
-  "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 1.253968253968254,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +9,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 153.28125,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 5.2990899085998535,
-      "kl": 2.7869672989472747,
-      "learning_rate": 5e-07,
-      "loss": 0.0028,
-      "reward": 2.3527547270059586,
-      "reward_std": 1.8630640702322125,
-      "rewards/concensus_correctness_reward_func": 0.4193124957382679,
-      "rewards/consensus_reward_func": 0.4375,
+      "completion_length": 386.78125,
+      "epoch": 0.12698412698412698,
+      "grad_norm": 2.824361801147461,
+      "kl": 0.0,
+      "learning_rate": 4.965903258506806e-07,
+      "loss": 0.0,
+      "reward": 2.1333778649568558,
+      "reward_std": 2.3683372139930725,
+      "rewards/concensus_correctness_reward_func": 0.3265624977648258,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.28844223637133837,
+      "rewards/final_correctness_reward_func": 0.6875,
+      "rewards/question_recreation_reward_func": 0.5482527948915958,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 0.9418749958276749,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.25856249686330557,
       "step": 2
     },
     {
-      "completion_length": 154.91666666666666,
-      "epoch": 1.0,
-      "grad_norm": 3.5864670276641846,
-      "kl": 4.980566471815109,
-      "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0037,
-      "reward": 4.6335459152857466,
-      "reward_std": 3.1853103240331015,
-      "rewards/concensus_correctness_reward_func": 2.1456666638453803,
-      "rewards/consensus_reward_func": 0.5,
+      "completion_length": 372.5,
+      "epoch": 0.25396825396825395,
+      "grad_norm": 2.9284257888793945,
+      "kl": 0.006983029292314313,
+      "learning_rate": 4.698684378016222e-07,
+      "loss": 0.0,
+      "reward": 2.490442905575037,
+      "reward_std": 3.8906576335430145,
+      "rewards/concensus_correctness_reward_func": 1.489624995738268,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.467712605992953,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.47975537180900574,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.16666666666666666,
-      "rewards/xmlcount_reward_func": 1.020166685183843,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.13043749844655395,
       "step": 4
     },
     {
-      "completion_length": 145.25,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 4.505063056945801,
-      "kl": 0.7166866697371006,
-      "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0007,
-      "reward": 2.0439836978912354,
-      "reward_std": 1.107317503541708,
-      "rewards/concensus_correctness_reward_func": 0.23931249603629112,
-      "rewards/consensus_reward_func": 0.25,
+      "completion_length": 383.28125,
+      "epoch": 0.38095238095238093,
+      "grad_norm": 3.603092670440674,
+      "kl": 0.014241222466807812,
+      "learning_rate": 4.193203929064353e-07,
+      "loss": 0.0,
+      "reward": 2.5441960506141186,
+      "reward_std": 1.947270791977644,
+      "rewards/concensus_correctness_reward_func": 1.9351250007748604,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.39148367382586,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3278835453093052,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 0.9288125038146973,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2811874996405095,
       "step": 6
     },
     {
-      "completion_length": 238.66666666666666,
-      "epoch": 2.0,
-      "grad_norm": 2.075845956802368,
-      "kl": 0.7329303125540415,
-      "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0005,
-      "reward": 2.372965137163798,
-      "reward_std": 1.550735956678788,
-      "rewards/concensus_correctness_reward_func": 0.40666666626930237,
+      "completion_length": 405.0,
+      "epoch": 0.5079365079365079,
+      "grad_norm": 1.8461979627609253,
+      "kl": 0.0014240605305531062,
+      "learning_rate": 3.5042385616324236e-07,
+      "loss": 0.0,
+      "reward": 3.485063388943672,
+      "reward_std": 5.036032758653164,
+      "rewards/concensus_correctness_reward_func": 2.0781250074505806,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5066318511962891,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.47100086882710457,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 0.7721666594346365,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.31093750218860805,
       "step": 8
     },
     {
-      "completion_length": 160.53125,
-      "epoch": 2.571428571428571,
-      "grad_norm": 4.525125980377197,
-      "kl": 0.6261784266680479,
-      "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0006,
-      "reward": 2.6195296347141266,
-      "reward_std": 0.8842782564461231,
-      "rewards/concensus_correctness_reward_func": 0.4779999926686287,
-      "rewards/consensus_reward_func": 0.5,
+      "completion_length": 387.75,
+      "epoch": 0.6349206349206349,
+      "grad_norm": 1.770258903503418,
+      "kl": 0.001492402094299905,
+      "learning_rate": 2.706448363680831e-07,
+      "loss": 0.0,
+      "reward": 1.7705548703670502,
+      "reward_std": 1.7027080208063126,
+      "rewards/concensus_correctness_reward_func": 0.33112500235438347,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3949983473867178,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 0.9652812406420708,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5824299026280642,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.26325000263750553,
       "step": 10
     },
     {
-      "completion_length": 130.25,
-      "epoch": 3.0,
-      "grad_norm": 2.9784047603607178,
-      "kl": 1.3293209373950958,
-      "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.001,
-      "reward": 2.8227996031443277,
-      "reward_std": 1.8227168321609497,
-      "rewards/concensus_correctness_reward_func": 0.40349999566872913,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "completion_length": 360.5625,
+      "epoch": 0.7619047619047619,
+      "grad_norm": 1.8140631914138794,
+      "kl": 0.002115250419592485,
+      "learning_rate": 1.886286282148002e-07,
+      "loss": 0.0,
+      "reward": 1.3889771271497011,
+      "reward_std": 2.282945653423667,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.499132938683033,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.21707089385017753,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 1.065999964872996,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.0155937559902668,
       "step": 12
     },
     {
-      "completion_length": 174.3125,
-      "epoch": 3.571428571428571,
-      "grad_norm": 4.740441799163818,
-      "kl": 2.012233894318342,
-      "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.002,
-      "reward": 2.627810388803482,
-      "reward_std": 1.9040176682174206,
-      "rewards/concensus_correctness_reward_func": 0.4789999909698963,
-      "rewards/consensus_reward_func": 0.5,
+      "completion_length": 381.1875,
+      "epoch": 0.8888888888888888,
+      "grad_norm": 1.5677131414413452,
+      "kl": 0.0031080194748938084,
+      "learning_rate": 1.1326296046939333e-07,
+      "loss": 0.0,
+      "reward": 1.264260726980865,
+      "reward_std": 2.3315915875136852,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5225291028618813,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3055732287466526,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 0.9387812465429306,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.14618749544024467,
       "step": 14
     },
     {
-      "completion_length": 178.79166666666666,
-      "epoch": 4.0,
-      "grad_norm": 3.826171875,
-      "kl": 0.6854466497898102,
-      "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0005,
-      "reward": 2.588684986035029,
-      "reward_std": 2.0806268056233725,
-      "rewards/concensus_correctness_reward_func": 0.48358333110809326,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "completion_length": 358.0,
+      "epoch": 1.0,
+      "grad_norm": 1.4784895181655884,
+      "kl": 0.0018846127038289393,
+      "learning_rate": 5.271487265090163e-08,
+      "loss": 0.0,
+      "reward": 1.3978956469467707,
+      "reward_std": 1.6970072899545943,
+      "rewards/concensus_correctness_reward_func": 0.20392856853348867,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.2634350123504798,
+      "rewards/final_correctness_reward_func": 0.21428571428571427,
+      "rewards/question_recreation_reward_func": 0.5163599167551313,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.20833333333333334,
-      "rewards/xmlcount_reward_func": 0.8833333253860474,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.15975000389984675,
       "step": 16
     },
     {
-      "completion_length": 159.96875,
-      "epoch": 4.571428571428571,
-      "grad_norm": 4.512508392333984,
-      "kl": 0.8317952752113342,
-      "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0008,
-      "reward": 2.3576600253582,
-      "reward_std": 1.593156099319458,
-      "rewards/concensus_correctness_reward_func": 0.3597499951720238,
-      "rewards/consensus_reward_func": 0.375,
+      "completion_length": 346.96875,
+      "epoch": 1.126984126984127,
+      "grad_norm": 2.7965524196624756,
+      "kl": 0.005831140035297722,
+      "learning_rate": 1.3545689574841341e-08,
+      "loss": 0.0,
+      "reward": 0.9347187448292971,
+      "reward_std": 0.8669621888548136,
+      "rewards/concensus_correctness_reward_func": 0.026624999940395355,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5000037755817175,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.2984375134110451,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 0.9510312527418137,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4221562445163727,
       "step": 18
     },
     {
-      "completion_length": 167.625,
-      "epoch": 5.0,
-      "grad_norm": 3.4769790172576904,
-      "kl": 0.3938896507024765,
-      "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0003,
-      "reward": 3.4651020963986716,
-      "reward_std": 2.1178342749675116,
-      "rewards/concensus_correctness_reward_func": 0.8057500024636587,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "completion_length": 357.25,
+      "epoch": 1.253968253968254,
+      "grad_norm": 2.8318896293640137,
+      "kl": 0.003781461316975765,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "reward": 1.5376606844365597,
+      "reward_std": 2.5096378745511174,
+      "rewards/concensus_correctness_reward_func": 0.7051874995231628,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.4759770520031452,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.43634821847081184,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.20833333333333334,
-      "rewards/xmlcount_reward_func": 0.975041667620341,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.08362500462681055,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 1.253968253968254,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.0013065586477750911,
-      "train_runtime": 248.9001,
-      "train_samples_per_second": 1.286,
-      "train_steps_per_second": 0.08
+      "train_loss": 4.052198395099183e-06,
+      "train_runtime": 837.0301,
+      "train_samples_per_second": 0.382,
+      "train_steps_per_second": 0.024
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 2,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43c6bbb9b9564e16958e96ea0dbde8132b8ab288f88170d3816109d3d06a54ab
-size 6008
+oid sha256:cbc32a14087ad8c7cb7cdc5b0ce37400e204c9fd3f918abb3ac9380e7ba2b5c2
+size 5944