End of training

adapter_config.json

@@ -24,13 +24,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "q_proj",
-    "down_proj",
-    "o_proj",
     "gate_proj",
+    "v_proj",
+    "o_proj",
+    "down_proj",
+    "k_proj",
     "up_proj",
-    "v_proj"
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3c815616ec5028b0537f54e59d9e547436432020a35b4677e5b19555c6fb39d
+oid sha256:d0ae6b798bf87415a3459688f725f184c2f6681cfe9f7637b79e23ea8000aba7
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.3574872329845675e-06,
-    "train_runtime": 556.8039,
+    "train_loss": 3.340189959999407e-06,
+    "train_runtime": 713.2682,
     "train_samples": 14,
-    "train_samples_per_second": 0.575,
-    "train_steps_per_second": 0.036
+    "train_samples_per_second": 0.449,
+    "train_steps_per_second": 0.028
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.3574872329845675e-06,
-    "train_runtime": 556.8039,
+    "train_loss": 3.340189959999407e-06,
+    "train_runtime": 713.2682,
     "train_samples": 14,
-    "train_samples_per_second": 0.575,
-    "train_steps_per_second": 0.036
+    "train_samples_per_second": 0.449,
+    "train_steps_per_second": 0.028
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 376.46875,
+      "completion_length": 322.375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 5.831172943115234,
-      "kl": 0.0016995659170788713,
+      "grad_norm": 2.0871474742889404,
+      "kl": 0.001733000564854592,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.19062989857047796,
-      "reward_std": 0.4901722795329988,
+      "reward": 0.19888813886791468,
+      "reward_std": 0.6061888420954347,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.14916115067899227,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.17770064366050065,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.041468748822808266,
+      "rewards/xmlcount_reward_func": -0.041312504559755325,
       "step": 2
     },
     {
-      "completion_length": 383.75,
+      "completion_length": 319.7916666666667,
       "epoch": 1.0,
-      "grad_norm": 1.1496180295944214,
-      "kl": 0.003234204836189747,
+      "grad_norm": 7.83439302444458,
+      "kl": 0.003303032834082842,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.3091644725451865,
-      "reward_std": 4.299379505217075,
-      "rewards/concensus_correctness_reward_func": 1.6875833322604497,
+      "reward": 0.6839615454276403,
+      "reward_std": 0.759387363990148,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.2919561235855023,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.4025865321358045,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": -0.024541677286227543,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.11470832924048106,
       "step": 4
     },
     {
-      "completion_length": 388.71875,
+      "completion_length": 329.03125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 1.4118651151657104,
-      "kl": 0.0018975575803779066,
+      "grad_norm": 1.6433956623077393,
+      "kl": 0.0019522544171195477,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.5037212371826172,
-      "reward_std": 0.6270111557096243,
+      "reward": 0.5661174282431602,
+      "reward_std": 0.6675986871123314,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2846899963915348,
+      "rewards/question_recreation_reward_func": 0.26621117535978556,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.21903125569224358,
+      "rewards/xmlcount_reward_func": 0.2374062491580844,
       "step": 6
     },
     {
-      "completion_length": 346.7916666666667,
+      "completion_length": 301.4583333333333,
       "epoch": 2.0,
-      "grad_norm": 1.7771496772766113,
-      "kl": 0.002119466351966063,
+      "grad_norm": 2.511329412460327,
+      "kl": 0.003298065799754113,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.6297860915462176,
-      "reward_std": 0.6073776086171468,
+      "reward": 0.8205540465811888,
+      "reward_std": 0.7265187874436378,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.20774442702531815,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.3611373820652564,
+      "rewards/soft_format_reward_func": 0.020833333333333332,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3387083373963833,
+      "rewards/xmlcount_reward_func": 0.2719166638950507,
       "step": 8
     },
     {
-      "completion_length": 359.4375,
+      "completion_length": 365.125,
       "epoch": 2.571428571428571,
-      "grad_norm": 1.9362791776657104,
-      "kl": 0.0018504365434637293,
+      "grad_norm": 16.952173233032227,
+      "kl": 0.0019986442930530757,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.8756589982658625,
-      "reward_std": 2.1364941727370024,
-      "rewards/concensus_correctness_reward_func": 1.25,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.8687562858685851,
+      "reward_std": 0.9658274040557444,
+      "rewards/concensus_correctness_reward_func": 0.015687499195337296,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.32647155225276947,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.29691252298653126,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.23668750608339906,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.22803125670179725,
       "step": 10
     },
     {
-      "completion_length": 418.0833333333333,
+      "completion_length": 251.66666666666666,
       "epoch": 3.0,
-      "grad_norm": 0.9003493189811707,
-      "kl": 0.0031372185136812427,
+      "grad_norm": 5.618197917938232,
+      "kl": 0.016620360877520095,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.3121936470270157,
-      "reward_std": 0.7566546574234962,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.3864253281305234,
+      "reward_std": 0.6003361530601978,
+      "rewards/concensus_correctness_reward_func": 0.07883333166440327,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.18977697317798933,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.24213367452224097,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.039083331823349,
+      "rewards/xmlcount_reward_func": 0.06545833374063174,
       "step": 12
     },
     {
-      "completion_length": 339.09375,
+      "completion_length": 339.84375,
       "epoch": 3.571428571428571,
-      "grad_norm": 2.4924376010894775,
-      "kl": 0.0017189035133924335,
+      "grad_norm": 3.2044646739959717,
+      "kl": 0.001727611837850418,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.2501581143587828,
-      "reward_std": 1.8279798203147948,
-      "rewards/concensus_correctness_reward_func": 0.671875,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.6722104325890541,
+      "reward_std": 0.7775694923475385,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.39768937043845654,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.31621043663471937,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.05559375323355198,
+      "rewards/xmlcount_reward_func": 0.16850000619888306,
       "step": 14
     },
     {
-      "completion_length": 415.25,
+      "completion_length": 329.5833333333333,
       "epoch": 4.0,
-      "grad_norm": 1.3772577047348022,
-      "kl": 0.0015851734206080437,
+      "grad_norm": 61.69892501831055,
+      "kl": 0.008029639682111641,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.37241995086272556,
-      "reward_std": 0.34809857575843733,
-      "rewards/concensus_correctness_reward_func": 0.020833333333333332,
+      "reward": 0.2587374324599902,
+      "reward_std": 0.42218482370177907,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2090449444949627,
+      "rewards/question_recreation_reward_func": 0.1902374264318496,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1425416680673758,
+      "rewards/xmlcount_reward_func": 0.06849999849994977,
       "step": 16
     },
     {
-      "completion_length": 294.5,
+      "completion_length": 397.8125,
       "epoch": 4.571428571428571,
-      "grad_norm": 2.1668484210968018,
-      "kl": 0.0016285206511383876,
+      "grad_norm": 2.8574352264404297,
+      "kl": 0.0016905490410863422,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.7301723528653383,
-      "reward_std": 1.2300486527383327,
-      "rewards/concensus_correctness_reward_func": 0.0390625,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.201634617522359,
+      "reward_std": 1.5953592797741294,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.3222661269828677,
+      "rewards/question_recreation_reward_func": 0.3076971317641437,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.18134374544024467,
+      "rewards/xmlcount_reward_func": 0.14393749367445707,
       "step": 18
     },
     {
-      "completion_length": 470.9583333333333,
+      "completion_length": 368.25,
       "epoch": 5.0,
-      "grad_norm": 0.6609285473823547,
-      "kl": 0.009601137678449353,
+      "grad_norm": 0.9341862797737122,
+      "kl": 0.0011831384617835283,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.2443943793574969,
-      "reward_std": 0.7095495462417603,
+      "reward": 0.19556983901808658,
+      "reward_std": 0.3893953717003266,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.20172771935661635,
+      "rewards/question_recreation_reward_func": 0.14365316454010704,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.04266665130853653,
+      "rewards/xmlcount_reward_func": 0.051916664155821,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.3574872329845675e-06,
-      "train_runtime": 556.8039,
-      "train_samples_per_second": 0.575,
-      "train_steps_per_second": 0.036
+      "train_loss": 3.340189959999407e-06,
+      "train_runtime": 713.2682,
+      "train_samples_per_second": 0.449,
+      "train_steps_per_second": 0.028
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ff4359996eec8b7ce1abdb9e745854c7d833effad10582875af7dac9f1c54ff
+oid sha256:42041399e6950eaab10631968500920d3d8576f210551467e73bb4d8f0d10129
 size 6008