End of training

README.md

@@ -40,7 +40,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 
 - TRL: 0.15.2
 - Transformers: 4.48.2
-- Pytorch: 2.5.1+cu121
+- Pytorch: 2.5.1
 - Datasets: 3.6.0
 - Tokenizers: 0.21.1
 

adapter_config.json

@@ -24,13 +24,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
-    "up_proj",
-    "q_proj",
     "o_proj",
-    "v_proj",
     "down_proj",
-    "k_proj"
+    "up_proj",
+    "k_proj",
+    "gate_proj",
+    "v_proj",
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:619030f187f1a7ed66ae988b67627c6461f61d9ad09ec49864892babd1bae79f
+oid sha256:8cd6afa37d03b192852e3643a7269121507a7535742557a47b0ec0ac94420aa3
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.9364487659222505e-06,
-    "train_runtime": 628.8319,
-    "train_samples": 44,
-    "train_samples_per_second": 0.509,
-    "train_steps_per_second": 0.032
+    "train_loss": 2.546286020788102e-06,
+    "train_runtime": 864.6962,
+    "train_samples": 20,
+    "train_samples_per_second": 0.37,
+    "train_steps_per_second": 0.023
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.9364487659222505e-06,
-    "train_runtime": 628.8319,
-    "train_samples": 44,
-    "train_samples_per_second": 0.509,
-    "train_steps_per_second": 0.032
+    "train_loss": 2.546286020788102e-06,
+    "train_runtime": 864.6962,
+    "train_samples": 20,
+    "train_samples_per_second": 0.37,
+    "train_steps_per_second": 0.023
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.8181818181818183,
+  "epoch": 4.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -9,209 +9,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 372.25,
-      "epoch": 0.18181818181818182,
-      "grad_norm": 1.9300328493118286,
-      "kl": 0.002685736268176697,
+      "completion_length": 537.6875,
+      "epoch": 0.4,
+      "grad_norm": 1.4502513408660889,
+      "kl": 0.0045772524463245645,
       "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 1.9535507448017597,
-      "reward_std": 2.3369685858488083,
-      "rewards/concensus_correctness_reward_func": 0.75,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.4177430057898164,
+      "reward_std": 2.1470614364370704,
+      "rewards/concensus_correctness_reward_func": 0.7554374933242798,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.4927695244550705,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.25036793970502913,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.27328124828636646,
+      "rewards/xmlcount_reward_func": 0.03693749010562897,
       "step": 2
     },
     {
-      "completion_length": 343.03125,
-      "epoch": 0.36363636363636365,
-      "grad_norm": 3.323760747909546,
-      "kl": 0.0038706637133145705,
+      "completion_length": 468.53125,
+      "epoch": 0.8,
+      "grad_norm": 1.9520736932754517,
+      "kl": 0.001855591282946989,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 1.1179207004606724,
-      "reward_std": 0.9569476544857025,
-      "rewards/concensus_correctness_reward_func": 0.09912500157952309,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.3959203027188778,
+      "reward_std": 1.1887737782672048,
+      "rewards/concensus_correctness_reward_func": 0.03125,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.36013949010521173,
+      "rewards/question_recreation_reward_func": 0.2753265555948019,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.2680312469601631,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.16065625473856926,
       "step": 4
     },
     {
-      "completion_length": 275.65625,
-      "epoch": 0.5454545454545454,
-      "grad_norm": 1.9294134378433228,
-      "kl": 0.0021318396175047383,
+      "completion_length": 441.78125,
+      "epoch": 1.2,
+      "grad_norm": 1.735103726387024,
+      "kl": 0.0016164953267434612,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 1.1408963352441788,
-      "reward_std": 0.967832338064909,
-      "rewards/concensus_correctness_reward_func": 0.125,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.623163734562695,
+      "reward_std": 0.7843756377696991,
+      "rewards/concensus_correctness_reward_func": 0.06006250157952309,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.4269276261329651,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.24881997099146247,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3858437556773424,
+      "rewards/xmlcount_reward_func": 0.06428124755620956,
       "step": 6
     },
     {
-      "completion_length": 319.96875,
-      "epoch": 0.7272727272727273,
-      "grad_norm": 1.2534875869750977,
-      "kl": 0.0027487699990160763,
+      "completion_length": 435.59375,
+      "epoch": 1.6,
+      "grad_norm": 2.2628138065338135,
+      "kl": 0.0013706091995118186,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 0.9228200912475586,
-      "reward_std": 1.5564873665571213,
-      "rewards/concensus_correctness_reward_func": 0.125,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.3420835845172405,
+      "reward_std": 0.797295784112066,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.3794763386249542,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.3639585729688406,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10584374889731407,
+      "rewards/xmlcount_reward_func": -0.08437500428408384,
       "step": 8
     },
     {
-      "completion_length": 435.375,
-      "epoch": 0.9090909090909091,
-      "grad_norm": 1.2784631252288818,
-      "kl": 0.002754525456111878,
+      "completion_length": 319.5625,
+      "epoch": 2.0,
+      "grad_norm": 2.1416268348693848,
+      "kl": 0.0025234216154785827,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 0.8444329872727394,
-      "reward_std": 1.0790864313021302,
-      "rewards/concensus_correctness_reward_func": 0.125,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.0948227606713772,
+      "reward_std": 1.085303759202361,
+      "rewards/concensus_correctness_reward_func": 0.08193749934434891,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.28580794762820005,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.37897900864481926,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1055000051856041,
+      "rewards/xmlcount_reward_func": 0.38390624336898327,
       "step": 10
     },
     {
-      "completion_length": 383.21875,
-      "epoch": 1.0909090909090908,
-      "grad_norm": 3.725076913833618,
-      "kl": 0.0038445138634415343,
+      "completion_length": 525.09375,
+      "epoch": 2.4,
+      "grad_norm": 2.21820068359375,
+      "kl": 0.001763832857250236,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 1.3737196251749992,
-      "reward_std": 1.564310446381569,
-      "rewards/concensus_correctness_reward_func": 0.24268750101327896,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 0.4163860874250531,
+      "reward_std": 1.0169010870158672,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.37809463776648045,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.413198578171432,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2529374994337559,
+      "rewards/xmlcount_reward_func": -0.0593124907463789,
       "step": 12
     },
     {
-      "completion_length": 408.4375,
-      "epoch": 1.2727272727272727,
-      "grad_norm": 3.1979212760925293,
-      "kl": 0.002077052035019733,
+      "completion_length": 311.84375,
+      "epoch": 2.8,
+      "grad_norm": 2.4503183364868164,
+      "kl": 0.0024037769326241687,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 4.569394081830978,
-      "reward_std": 5.111470676958561,
-      "rewards/concensus_correctness_reward_func": 3.208000000566244,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.1352357491850853,
+      "reward_std": 1.1741696512326598,
+      "rewards/concensus_correctness_reward_func": 0.15131249651312828,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.46067542023956776,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.37954823300242424,
       "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.19759375415742397,
+      "rewards/xmlcount_reward_func": 0.2762500010430813,
       "step": 14
     },
     {
-      "completion_length": 436.34375,
-      "epoch": 1.4545454545454546,
-      "grad_norm": 1.4435375928878784,
-      "kl": 0.0014565634628525004,
+      "completion_length": 371.1875,
+      "epoch": 3.2,
+      "grad_norm": 2.3564932346343994,
+      "kl": 0.003548893568222411,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 1.052122674882412,
-      "reward_std": 0.8749217577278614,
-      "rewards/concensus_correctness_reward_func": 0.03125,
+      "reward": 0.6700164806097746,
+      "reward_std": 0.8069747863337398,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.4829976772889495,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3270789897069335,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4753750041127205,
+      "rewards/xmlcount_reward_func": 0.2179374946281314,
       "step": 16
     },
     {
-      "completion_length": 434.21875,
-      "epoch": 1.6363636363636362,
-      "grad_norm": 1.8603307008743286,
-      "kl": 0.004560476292681415,
+      "completion_length": 364.21875,
+      "epoch": 3.6,
+      "grad_norm": 3.2016563415527344,
+      "kl": 0.0021984638296999037,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 1.5926124211400747,
-      "reward_std": 2.343985839514062,
-      "rewards/concensus_correctness_reward_func": 0.65625,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.9208500199019909,
+      "reward_std": 1.1289626825600863,
+      "rewards/concensus_correctness_reward_func": 0.21381250023841858,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.3674874210264534,
+      "rewards/question_recreation_reward_func": 0.184193748049438,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.2407499998807907,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2728437576442957,
       "step": 18
     },
     {
-      "completion_length": 310.65625,
-      "epoch": 1.8181818181818183,
-      "grad_norm": 1.8424596786499023,
-      "kl": 0.0032100926473503932,
+      "completion_length": 364.34375,
+      "epoch": 4.0,
+      "grad_norm": 2.2764692306518555,
+      "kl": 0.0035686935880221426,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 0.7539666602388024,
-      "reward_std": 0.9992944076657295,
-      "rewards/concensus_correctness_reward_func": 0.03506249934434891,
+      "reward": 1.4730796124786139,
+      "reward_std": 1.3868739902973175,
+      "rewards/concensus_correctness_reward_func": 0.32718750089406967,
       "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.31849788781255484,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.46451712027192116,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.25978125259280205,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3688750071451068,
       "step": 20
     },
     {
-      "epoch": 1.8181818181818183,
+      "epoch": 4.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.9364487659222505e-06,
-      "train_runtime": 628.8319,
-      "train_samples_per_second": 0.509,
-      "train_steps_per_second": 0.032
+      "train_loss": 2.546286020788102e-06,
+      "train_runtime": 864.6962,
+      "train_samples_per_second": 0.37,
+      "train_steps_per_second": 0.023
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 2,
+  "num_train_epochs": 4,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {