End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93e137963d8b647770c17743af9c890ab09dc1d18c9cd4b73418178f5ef353ed
+oid sha256:31e5f7f03a6afa6625fa34b93901fd151d9b4782bc1869811e82f83cf0dddd6f
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.914579054049682e-05,
-    "train_runtime": 304.4927,
+    "train_loss": 2.633521071402356e-05,
+    "train_runtime": 297.393,
     "train_samples": 14,
-    "train_samples_per_second": 1.051,
-    "train_steps_per_second": 0.066
+    "train_samples_per_second": 1.076,
+    "train_steps_per_second": 0.067
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.914579054049682e-05,
-    "train_runtime": 304.4927,
+    "train_loss": 2.633521071402356e-05,
+    "train_runtime": 297.393,
     "train_samples": 14,
-    "train_samples_per_second": 1.051,
-    "train_steps_per_second": 0.066
+    "train_samples_per_second": 1.076,
+    "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 218.4375,
+      "completion_length": 249.21875,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.6667295694351196,
-      "kl": 0.026695957989431918,
+      "grad_norm": 0.7512394785881042,
+      "kl": 0.026906487299129367,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 7.303258091211319,
-      "reward_std": 6.162659838795662,
-      "rewards/concensus_correctness_reward_func": 4.495250031352043,
-      "rewards/consensus_reward_func": 0.9375,
+      "reward": 7.222172915935516,
+      "reward_std": 5.19085992872715,
+      "rewards/concensus_correctness_reward_func": 4.511687513440847,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.4684457331895828,
+      "rewards/final_correctness_reward_func": 1.125,
+      "rewards/question_recreation_reward_func": 0.38086048141121864,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.542687501758337,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5796250104904175,
       "step": 2
     },
     {
-      "completion_length": 204.91666666666666,
+      "completion_length": 201.41666666666666,
       "epoch": 1.0,
-      "grad_norm": 0.6115171909332275,
-      "kl": 0.025265894054124754,
+      "grad_norm": 0.45711734890937805,
+      "kl": 0.026981870643794537,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 8.634976267814636,
-      "reward_std": 7.231328090031941,
-      "rewards/concensus_correctness_reward_func": 5.156750018397967,
-      "rewards/consensus_reward_func": 0.75,
+      "reward": 6.3005698919296265,
+      "reward_std": 4.253052691618602,
+      "rewards/concensus_correctness_reward_func": 3.2181666493415833,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 1.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.46435124178727466,
+      "rewards/question_recreation_reward_func": 0.6594448884328207,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.041666666666666664,
-      "rewards/xmlcount_reward_func": 0.8055416742960612,
+      "rewards/xmlcount_reward_func": 0.7146250009536743,
       "step": 4
     },
     {
-      "completion_length": 230.375,
+      "completion_length": 206.65625,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.8700223565101624,
-      "kl": 0.06901322270277888,
+      "grad_norm": 1.5178693532943726,
+      "kl": 0.03174284310080111,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0001,
-      "reward": 5.524010732769966,
-      "reward_std": 5.278677627444267,
-      "rewards/concensus_correctness_reward_func": 2.9027500078082085,
-      "rewards/consensus_reward_func": 0.5625,
+      "loss": 0.0,
+      "reward": 8.552556872367859,
+      "reward_std": 7.061087839305401,
+      "rewards/concensus_correctness_reward_func": 5.843437597155571,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.4469796493649483,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.37958788871765137,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.7992812357842922,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.6107812561094761,
       "step": 6
     },
     {
-      "completion_length": 269.9166666666667,
+      "completion_length": 211.125,
       "epoch": 2.0,
-      "grad_norm": 0.7686216235160828,
-      "kl": 0.017645259698232014,
+      "grad_norm": 0.6583097577095032,
+      "kl": 0.027615504804998636,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 6.857015669345856,
-      "reward_std": 5.87060546875,
-      "rewards/concensus_correctness_reward_func": 3.48191666106383,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 10.130588610967001,
+      "reward_std": 9.325920085112253,
+      "rewards/concensus_correctness_reward_func": 6.703166683514913,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.6566405842701594,
+      "rewards/final_correctness_reward_func": 1.4166666666666667,
+      "rewards/question_recreation_reward_func": 0.5063388552516699,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.4892916704217593,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.6710833311080933,
       "step": 8
     },
     {
-      "completion_length": 206.40625,
+      "completion_length": 217.78125,
       "epoch": 2.571428571428571,
-      "grad_norm": 1.030617594718933,
-      "kl": 0.0330960585270077,
+      "grad_norm": 1.0431419610977173,
+      "kl": 0.03605657909065485,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 4.651839181780815,
-      "reward_std": 3.271824397146702,
-      "rewards/concensus_correctness_reward_func": 2.158562485128641,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 6.483449876308441,
+      "reward_std": 6.295221641659737,
+      "rewards/concensus_correctness_reward_func": 3.76581247523427,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.5593391647562385,
+      "rewards/final_correctness_reward_func": 1.25,
+      "rewards/question_recreation_reward_func": 0.5397312082350254,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.7620625272393227,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.552906259894371,
       "step": 10
     },
     {
-      "completion_length": 218.54166666666666,
+      "completion_length": 228.75,
       "epoch": 3.0,
-      "grad_norm": 0.6721370220184326,
-      "kl": 0.029874651227146387,
+      "grad_norm": 0.6969460844993591,
+      "kl": 0.031089524428049724,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 8.552127718925476,
-      "reward_std": 7.366494715213776,
-      "rewards/concensus_correctness_reward_func": 5.438833256562551,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 6.5001393755276995,
+      "reward_std": 6.519809847076734,
+      "rewards/concensus_correctness_reward_func": 4.256999999284744,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.30829449991385144,
+      "rewards/final_correctness_reward_func": 0.5833333333333334,
+      "rewards/question_recreation_reward_func": 0.36276430636644363,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.6175000071525574,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.713708351055781,
       "step": 12
     },
     {
-      "completion_length": 220.875,
+      "completion_length": 198.9375,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.7800900340080261,
-      "kl": 0.02692935918457806,
+      "grad_norm": 0.8872530460357666,
+      "kl": 0.02684053615666926,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 7.269978225231171,
-      "reward_std": 6.192141018807888,
-      "rewards/concensus_correctness_reward_func": 4.787062469869852,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 5.913686141371727,
+      "reward_std": 4.713027611374855,
+      "rewards/concensus_correctness_reward_func": 2.8759999722242355,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.6210407689213753,
+      "rewards/final_correctness_reward_func": 1.3125,
+      "rewards/question_recreation_reward_func": 0.5502487141638994,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.7056250050663948,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.7530625071376562,
       "step": 14
     },
     {
-      "completion_length": 232.70833333333334,
+      "completion_length": 221.125,
       "epoch": 4.0,
-      "grad_norm": 1.0308959484100342,
-      "kl": 0.02224013830224673,
+      "grad_norm": 0.5982879400253296,
+      "kl": 0.029973368781308334,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 5.702548424402873,
-      "reward_std": 4.750810205936432,
-      "rewards/concensus_correctness_reward_func": 3.1970000167687735,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 5.973727345466614,
+      "reward_std": 3.197078049182892,
+      "rewards/concensus_correctness_reward_func": 3.357833360632261,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0833333333333333,
-      "rewards/question_recreation_reward_func": 0.25150686626633006,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.4214770123362541,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7540416618188223,
+      "rewards/strict_format_reward_func": 0.041666666666666664,
+      "rewards/xmlcount_reward_func": 0.5694166744748751,
       "step": 16
     },
     {
-      "completion_length": 219.8125,
+      "completion_length": 243.5,
       "epoch": 4.571428571428571,
-      "grad_norm": 1.084661841392517,
-      "kl": 0.04743693000636995,
+      "grad_norm": 1.070564866065979,
+      "kl": 0.031115309335291386,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 6.552321285009384,
-      "reward_std": 6.906356409192085,
-      "rewards/concensus_correctness_reward_func": 3.8233124800026417,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 6.433173283934593,
+      "reward_std": 6.740131661295891,
+      "rewards/concensus_correctness_reward_func": 3.5397499799728394,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.34860267490148544,
+      "rewards/final_correctness_reward_func": 1.1875,
+      "rewards/question_recreation_reward_func": 0.5655796639621258,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.7554062642157078,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.7028437554836273,
       "step": 18
     },
     {
-      "completion_length": 195.75,
+      "completion_length": 209.5,
       "epoch": 5.0,
-      "grad_norm": 1.2956124544143677,
-      "kl": 0.022885148258258898,
+      "grad_norm": 0.7200393676757812,
+      "kl": 0.03196889193107685,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 4.439935863018036,
-      "reward_std": 2.670658846696218,
-      "rewards/concensus_correctness_reward_func": 2.021749973297119,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 8.892809788386026,
+      "reward_std": 8.186325470606485,
+      "rewards/concensus_correctness_reward_func": 5.795250087976456,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.4781026641527812,
+      "rewards/final_correctness_reward_func": 1.25,
+      "rewards/question_recreation_reward_func": 0.44406017164389294,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6900833398103714,
+      "rewards/xmlcount_reward_func": 0.5701666797200838,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.914579054049682e-05,
-      "train_runtime": 304.4927,
-      "train_samples_per_second": 1.051,
-      "train_steps_per_second": 0.066
+      "train_loss": 2.633521071402356e-05,
+      "train_runtime": 297.393,
+      "train_samples_per_second": 1.076,
+      "train_steps_per_second": 0.067
     }
   ],
   "logging_steps": 2,