End of training

adapter_config.json

@@ -24,13 +24,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
-    "v_proj",
-    "up_proj",
     "gate_proj",
-    "k_proj",
     "down_proj",
-    "o_proj"
+    "q_proj",
+    "up_proj",
+    "o_proj",
+    "v_proj",
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea7e4dff1a8ec2157cfcfc2c92f38aa3272f6cc59a065733ae70cd121dd592c4
+oid sha256:d5a5db1ca33542874d8002e36eb5419408164060f93f435109f44e4e741a65be
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.4771618452869006e-06,
-    "train_runtime": 2220.5118,
-    "train_samples": 11,
-    "train_samples_per_second": 0.144,
+    "train_loss": 2.6350203826908115e-06,
+    "train_runtime": 2184.1674,
+    "train_samples": 22,
+    "train_samples_per_second": 0.147,
     "train_steps_per_second": 0.009
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.4771618452869006e-06,
-    "train_runtime": 2220.5118,
-    "train_samples": 11,
-    "train_samples_per_second": 0.144,
+    "train_loss": 2.6350203826908115e-06,
+    "train_runtime": 2184.1674,
+    "train_samples": 22,
+    "train_samples_per_second": 0.147,
     "train_steps_per_second": 0.009
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 6.7272727272727275,
+  "epoch": 3.3636363636363638,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 382.375,
-      "epoch": 0.7272727272727273,
-      "grad_norm": 4.245586395263672,
-      "kl": 0.0019058704710914753,
+      "completion_length": 459.4375,
+      "epoch": 0.36363636363636365,
+      "grad_norm": 1.3302592039108276,
+      "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 0.4013321031816304,
-      "reward_std": 0.5064181148773059,
+      "reward": 0.04405219480395317,
+      "reward_std": 0.6433118600398302,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.2303008483722806,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2288334424374625,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10853125154972076,
+      "rewards/xmlcount_reward_func": -0.18478125147521496,
       "step": 2
     },
     {
-      "completion_length": 330.64285714285717,
-      "epoch": 1.3636363636363638,
-      "grad_norm": 1.9667288064956665,
-      "kl": 0.002137252836421664,
+      "completion_length": 409.90625,
+      "epoch": 0.7272727272727273,
+      "grad_norm": 1.2176474332809448,
+      "kl": 0.0020845978724537417,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 0.13177615031599998,
-      "reward_std": 0.10159020511699575,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.7678184271790087,
+      "reward_std": 0.7591042327694595,
+      "rewards/concensus_correctness_reward_func": 0.013437500223517418,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11009758045630795,
+      "rewards/question_recreation_reward_func": 0.26688094018027186,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.02167857119015285,
+      "rewards/xmlcount_reward_func": 0.11249999701976776,
       "step": 4
     },
     {
-      "completion_length": 441.60714285714283,
-      "epoch": 2.0,
-      "grad_norm": 1.6228898763656616,
-      "kl": 0.001162897820384907,
+      "completion_length": 367.9583333333333,
+      "epoch": 1.0,
+      "grad_norm": 1.5351356267929077,
+      "kl": 0.004026121634524316,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 0.12548786721059255,
-      "reward_std": 0.211075826681086,
+      "reward": 0.5474015871683756,
+      "reward_std": 0.8145603885253271,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1422378732157605,
+      "rewards/question_recreation_reward_func": 0.3452765668431918,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.016750001614647254,
+      "rewards/xmlcount_reward_func": 0.11879166960716248,
       "step": 6
     },
     {
-      "completion_length": 491.40625,
-      "epoch": 2.7272727272727275,
-      "grad_norm": 2.9933621883392334,
-      "kl": 0.00119662764336681,
+      "completion_length": 340.90625,
+      "epoch": 1.3636363636363638,
+      "grad_norm": 2.2870802879333496,
+      "kl": 0.0028898825694341213,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 0.09560761321336031,
-      "reward_std": 0.22292510606348515,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.7129990463145077,
+      "reward_std": 1.1731511121615767,
+      "rewards/concensus_correctness_reward_func": 0.013437500223517418,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.12760760867968202,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.25137403002008796,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.03199999779462814,
+      "rewards/xmlcount_reward_func": 0.010687506292015314,
       "step": 8
     },
     {
-      "completion_length": 395.0,
-      "epoch": 3.3636363636363638,
-      "grad_norm": 2.040949583053589,
-      "kl": 0.0011819826738376702,
+      "completion_length": 415.125,
+      "epoch": 1.7272727272727273,
+      "grad_norm": 5.562981605529785,
+      "kl": 0.0016170966700883582,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 0.12203995590763432,
-      "reward_std": 0.23508733504318766,
+      "reward": 0.295339772477746,
+      "reward_std": 0.6460105914156884,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.12200423888862133,
+      "rewards/question_recreation_reward_func": 0.27162102051079273,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 3.571595464433942e-05,
+      "rewards/xmlcount_reward_func": 0.023718755692243576,
       "step": 10
     },
     {
-      "completion_length": 397.7857142857143,
-      "epoch": 4.0,
-      "grad_norm": 2.2787046432495117,
-      "kl": 0.0015667497978678771,
+      "completion_length": 523.875,
+      "epoch": 2.0,
+      "grad_norm": 1.5765122175216675,
+      "kl": 0.0012822053298198928,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 0.20091228266911848,
-      "reward_std": 0.2962765635124275,
+      "reward": 0.5160771167526642,
+      "reward_std": 0.8615054544061422,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1388408448547125,
+      "rewards/question_recreation_reward_func": 0.1667854206946989,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06207142770290375,
+      "rewards/xmlcount_reward_func": 0.01595833556105693,
       "step": 12
     },
     {
-      "completion_length": 415.34375,
-      "epoch": 4.7272727272727275,
-      "grad_norm": 2.0091869831085205,
-      "kl": 0.0016181922619580291,
+      "completion_length": 423.1875,
+      "epoch": 2.3636363636363638,
+      "grad_norm": 3.4827518463134766,
+      "kl": 0.004907177797576878,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 0.15321381436660886,
-      "reward_std": 0.34077310003340244,
+      "reward": 0.1884107068181038,
+      "reward_std": 0.3662752130185254,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.15933882864192128,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.12262945214752108,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.006125002633780241,
+      "rewards/xmlcount_reward_func": 0.050156254321336746,
       "step": 14
     },
     {
-      "completion_length": 472.82142857142856,
-      "epoch": 5.363636363636363,
-      "grad_norm": 1.0329787731170654,
-      "kl": 0.000985492827437286,
+      "completion_length": 346.875,
+      "epoch": 2.7272727272727275,
+      "grad_norm": 4.646650314331055,
+      "kl": 0.006251230763155036,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 0.07800916742001261,
-      "reward_std": 0.23973628639110497,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.8924960913136601,
+      "reward_std": 1.0580921154469252,
+      "rewards/concensus_correctness_reward_func": 0.05974999815225601,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.14804488367268018,
+      "rewards/question_recreation_reward_func": 0.264589830301702,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.07003571412393025,
+      "rewards/xmlcount_reward_func": 0.0681562414392829,
       "step": 16
     },
     {
-      "completion_length": 347.64285714285717,
-      "epoch": 6.0,
-      "grad_norm": 1.7792203426361084,
-      "kl": 0.0016959133291883127,
+      "completion_length": 447.875,
+      "epoch": 3.0,
+      "grad_norm": 2.753358840942383,
+      "kl": 0.0035982539411634207,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 0.14558894453304155,
-      "reward_std": 0.31389917459871086,
+      "reward": 0.31809323467314243,
+      "reward_std": 0.6482827632377545,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.21908894726740463,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.2697598912442724,
+      "rewards/soft_format_reward_func": 0.020833333333333332,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.07350000419786998,
+      "rewards/xmlcount_reward_func": -0.055833334724108376,
       "step": 18
     },
     {
-      "completion_length": 435.21875,
-      "epoch": 6.7272727272727275,
-      "grad_norm": 1.950781226158142,
-      "kl": 0.0024183838177123107,
+      "completion_length": 371.9375,
+      "epoch": 3.3636363636363638,
+      "grad_norm": 37.19569778442383,
+      "kl": 0.0018885131285060197,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 0.5787201011553407,
-      "reward_std": 0.6228024321608245,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.9909486873075366,
+      "reward_std": 1.7817546646110713,
+      "rewards/concensus_correctness_reward_func": 0.625,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.2445013471879065,
+      "rewards/question_recreation_reward_func": 0.15935491654090583,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2092187525704503,
+      "rewards/xmlcount_reward_func": 0.08159375563263893,
       "step": 20
     },
     {
-      "epoch": 6.7272727272727275,
+      "epoch": 3.3636363636363638,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.4771618452869006e-06,
-      "train_runtime": 2220.5118,
-      "train_samples_per_second": 0.144,
+      "train_loss": 2.6350203826908115e-06,
+      "train_runtime": 2184.1674,
+      "train_samples_per_second": 0.147,
       "train_steps_per_second": 0.009
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 10,
+  "num_train_epochs": 4,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40da36acc51b0694a52552ca44416433ff9dc04afbe8c57af271ea91a2ff1d1a
+oid sha256:8725a6f3404d2eb26f02602cd7874bf6057118871a66f864c79f96d74aabb209
 size 6072