End of training

README.md

@@ -40,8 +40,8 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 
 - TRL: 0.15.2
 - Transformers: 4.51.3
-- Pytorch: 2.6.0
-- Datasets: 3.5.0
+- Pytorch: 2.5.1
+- Datasets: 3.6.0
 - Tokenizers: 0.21.1
 
 ## Citations

adapter_config.json

@@ -24,13 +24,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "up_proj",
-    "o_proj",
+    "down_proj",
+    "q_proj",
     "k_proj",
+    "o_proj",
+    "v_proj",
     "gate_proj",
-    "down_proj",
-    "q_proj"
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:058c55c27529f6256af32deef6d30cb8bee9fad1f96ecb4aab320327df3d29df
+oid sha256:f1b22e7ff1c52bf61a1a162805358c06390056bac13767a24daf26d74e5d1d76
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.846263126026315e-06,
-    "train_runtime": 1130.8008,
-    "train_samples": 14,
-    "train_samples_per_second": 0.283,
-    "train_steps_per_second": 0.018
+    "train_loss": 6.33447747873106e-06,
+    "train_runtime": 621.3953,
+    "train_samples": 32,
+    "train_samples_per_second": 0.515,
+    "train_steps_per_second": 0.032
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.846263126026315e-06,
-    "train_runtime": 1130.8008,
-    "train_samples": 14,
-    "train_samples_per_second": 0.283,
-    "train_steps_per_second": 0.018
+    "train_loss": 6.33447747873106e-06,
+    "train_runtime": 621.3953,
+    "train_samples": 32,
+    "train_samples_per_second": 0.515,
+    "train_steps_per_second": 0.032
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 2.5,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 394.46875,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 1.2339857816696167,
-      "kl": 0.0014976889069657773,
+      "completion_length": 379.3125,
+      "epoch": 0.25,
+      "grad_norm": 2.7439236640930176,
+      "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.4177959216758609,
-      "reward_std": 0.47653236147016287,
-      "rewards/concensus_correctness_reward_func": 0.0078125,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 3.3803121647797525,
+      "reward_std": 3.566028727334924,
+      "rewards/concensus_correctness_reward_func": 1.875,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2874209163710475,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.3854058999568224,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.12256250437349081,
+      "rewards/xmlcount_reward_func": 0.3074062436353415,
       "step": 2
     },
     {
-      "completion_length": 401.75,
-      "epoch": 1.0,
-      "grad_norm": 0.9314263463020325,
-      "kl": 0.0014763531604936968,
+      "completion_length": 248.9375,
+      "epoch": 0.5,
+      "grad_norm": 7.657437801361084,
+      "kl": 0.0056438162791891955,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.8192004313071569,
-      "reward_std": 0.9109265580773354,
-      "rewards/concensus_correctness_reward_func": 0.054499998688697815,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.7027008049190044,
+      "reward_std": 2.5414942312054336,
+      "rewards/concensus_correctness_reward_func": 0.6851250007748604,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.35807543123761815,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.3183257170021534,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07329167673985164,
+      "rewards/xmlcount_reward_func": 0.13675000239163637,
       "step": 4
     },
     {
-      "completion_length": 382.8125,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 10.93860912322998,
-      "kl": 0.0044528390862978995,
+      "completion_length": 320.71875,
+      "epoch": 0.75,
+      "grad_norm": 3.027006149291992,
+      "kl": 0.00369156566739548,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.4012568201869726,
-      "reward_std": 0.8289460856467485,
-      "rewards/concensus_correctness_reward_func": 0.02043749950826168,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.9596255305223167,
+      "reward_std": 1.2235960713587701,
+      "rewards/concensus_correctness_reward_func": 0.1197500005364418,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.331225560978055,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.19928179658018053,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.04959375876933336,
+      "rewards/xmlcount_reward_func": 0.20309375040233135,
       "step": 6
     },
     {
-      "completion_length": 431.2916666666667,
-      "epoch": 2.0,
-      "grad_norm": 0.7058228254318237,
-      "kl": 0.0016016751566591363,
+      "completion_length": 305.0,
+      "epoch": 1.0,
+      "grad_norm": 57.53217697143555,
+      "kl": 0.0027342141838744283,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 3.168473449846109,
-      "reward_std": 2.236310025677085,
-      "rewards/concensus_correctness_reward_func": 2.5,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.513476200401783,
+      "reward_std": 2.38468911498785,
+      "rewards/concensus_correctness_reward_func": 0.6851250007748604,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.312098595003287,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.34213246311992407,
+      "rewards/soft_format_reward_func": 0.03125,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1063750001291434,
+      "rewards/xmlcount_reward_func": 0.07996875513345003,
       "step": 8
     },
     {
-      "completion_length": 392.625,
-      "epoch": 2.571428571428571,
-      "grad_norm": 36.917015075683594,
-      "kl": 0.0015262642700690776,
+      "completion_length": 332.9375,
+      "epoch": 1.25,
+      "grad_norm": 52.287322998046875,
+      "kl": 0.037694583530537784,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.6821226291358471,
-      "reward_std": 0.5203963043168187,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.110419673845172,
+      "reward_std": 1.192315123975277,
+      "rewards/concensus_correctness_reward_func": 0.16950000077486038,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.3443101402372122,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.2483259281143546,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.08781249797903001,
+      "rewards/xmlcount_reward_func": 0.19259374681860209,
       "step": 10
     },
     {
-      "completion_length": 458.8333333333333,
-      "epoch": 3.0,
-      "grad_norm": 1.1164511442184448,
-      "kl": 0.0013929471994439762,
+      "completion_length": 353.40625,
+      "epoch": 1.5,
+      "grad_norm": 2.1427807807922363,
+      "kl": 0.0023543949209852144,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.5158853853742282,
-      "reward_std": 2.3134964207808175,
-      "rewards/concensus_correctness_reward_func": 0.8878333320220312,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.6070140902884305,
+      "reward_std": 2.405821370659396,
+      "rewards/concensus_correctness_reward_func": 0.7452500015497208,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.36738542715708417,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.358639074023813,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2606666684150696,
+      "rewards/xmlcount_reward_func": -0.05937499180436134,
       "step": 12
     },
     {
-      "completion_length": 400.53125,
-      "epoch": 3.571428571428571,
-      "grad_norm": 1.1706537008285522,
-      "kl": 0.0035119867679895833,
+      "completion_length": 258.75,
+      "epoch": 1.75,
+      "grad_norm": 25.269350051879883,
+      "kl": 0.002261648085550405,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.229899922851473,
-      "reward_std": 1.5985346003435552,
+      "reward": 1.369703633710742,
+      "reward_std": 2.061299598775804,
       "rewards/concensus_correctness_reward_func": 0.625,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.2692748848348856,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.2552348882891238,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0231250012293458,
+      "rewards/xmlcount_reward_func": 0.05196875333786011,
       "step": 14
     },
     {
-      "completion_length": 431.2916666666667,
-      "epoch": 4.0,
-      "grad_norm": 0.6544820070266724,
-      "kl": 0.0016228632496980329,
+      "completion_length": 400.3125,
+      "epoch": 2.0,
+      "grad_norm": 1.878326416015625,
+      "kl": 0.0017821879300754517,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.5711955279111862,
-      "reward_std": 0.5347563698887825,
-      "rewards/concensus_correctness_reward_func": 0.054499998688697815,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.3578586280345917,
+      "reward_std": 1.6749285701662302,
+      "rewards/concensus_correctness_reward_func": 0.19862500578165054,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.40036220972736675,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.32482736092060804,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.09549999609589577,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.2562812389805913,
       "step": 16
     },
     {
-      "completion_length": 454.53125,
-      "epoch": 4.571428571428571,
-      "grad_norm": 1.6570597887039185,
-      "kl": 0.0016838895244291052,
+      "completion_length": 388.875,
+      "epoch": 2.25,
+      "grad_norm": 2.714452028274536,
+      "kl": 0.0016746084875194356,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.6594347928185016,
-      "reward_std": 0.8347256639972329,
-      "rewards/concensus_correctness_reward_func": 0.06131249666213989,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.4144831225275993,
+      "reward_std": 1.6663512866944075,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.314653554931283,
+      "rewards/question_recreation_reward_func": 0.2610769057646394,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.158468761947006,
+      "rewards/xmlcount_reward_func": 0.21590624377131462,
       "step": 18
     },
     {
-      "completion_length": 351.5416666666667,
-      "epoch": 5.0,
-      "grad_norm": 0.903715193271637,
-      "kl": 0.0017104301368817687,
+      "completion_length": 361.03125,
+      "epoch": 2.5,
+      "grad_norm": 87.17727661132812,
+      "kl": 0.0055512295657536015,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.8093653731048107,
-      "reward_std": 0.8842336858312289,
-      "rewards/concensus_correctness_reward_func": 0.0625,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.9679193682968616,
+      "reward_std": 2.6846568696200848,
+      "rewards/concensus_correctness_reward_func": 0.6851250007748604,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.38882372714579105,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.44204438058659434,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10804166582723458,
+      "rewards/xmlcount_reward_func": 0.27824999764561653,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 2.5,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.846263126026315e-06,
-      "train_runtime": 1130.8008,
-      "train_samples_per_second": 0.283,
-      "train_steps_per_second": 0.018
+      "train_loss": 6.33447747873106e-06,
+      "train_runtime": 621.3953,
+      "train_samples_per_second": 0.515,
+      "train_steps_per_second": 0.032
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 3,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48b54a1e0de6d9a1bdb473dde2277a8d91adad179462aa76ef6aa443e4aba001
+oid sha256:51612f2a388e62289944201fdd697aed0e365b4469cd71472bfbc3aa87646c89
 size 6008