End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31e5f7f03a6afa6625fa34b93901fd151d9b4782bc1869811e82f83cf0dddd6f
+oid sha256:0c27540c74d1f19f5245c0bdc6785826e914b318b78aecb64dec4d312968bd42
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.633521071402356e-05,
-    "train_runtime": 297.393,
+    "train_loss": 2.5608179566916078e-05,
+    "train_runtime": 324.6357,
     "train_samples": 14,
-    "train_samples_per_second": 1.076,
-    "train_steps_per_second": 0.067
+    "train_samples_per_second": 0.986,
+    "train_steps_per_second": 0.062
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.633521071402356e-05,
-    "train_runtime": 297.393,
+    "train_loss": 2.5608179566916078e-05,
+    "train_runtime": 324.6357,
     "train_samples": 14,
-    "train_samples_per_second": 1.076,
-    "train_steps_per_second": 0.067
+    "train_samples_per_second": 0.986,
+    "train_steps_per_second": 0.062
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 249.21875,
+      "completion_length": 241.1875,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.7512394785881042,
-      "kl": 0.026906487299129367,
+      "grad_norm": 0.9085168242454529,
+      "kl": 0.023555907187983394,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 7.222172915935516,
-      "reward_std": 5.19085992872715,
-      "rewards/concensus_correctness_reward_func": 4.511687513440847,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 6.318546280264854,
+      "reward_std": 5.344433315098286,
+      "rewards/concensus_correctness_reward_func": 4.012687494978309,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.125,
-      "rewards/question_recreation_reward_func": 0.38086048141121864,
+      "rewards/final_correctness_reward_func": 0.6875,
+      "rewards/question_recreation_reward_func": 0.3027961915358901,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5796250104904175,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.6124374950304627,
       "step": 2
     },
     {
-      "completion_length": 201.41666666666666,
+      "completion_length": 246.20833333333334,
       "epoch": 1.0,
-      "grad_norm": 0.45711734890937805,
-      "kl": 0.026981870643794537,
+      "grad_norm": 0.575706422328949,
+      "kl": 0.03216843865811825,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 6.3005698919296265,
-      "reward_std": 4.253052691618602,
-      "rewards/concensus_correctness_reward_func": 3.2181666493415833,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 5.88692839940389,
+      "reward_std": 5.155879924694697,
+      "rewards/concensus_correctness_reward_func": 2.947500000397364,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.6594448884328207,
+      "rewards/final_correctness_reward_func": 1.25,
+      "rewards/question_recreation_reward_func": 0.5535533304015795,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.041666666666666664,
-      "rewards/xmlcount_reward_func": 0.7146250009536743,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.6150416483481725,
       "step": 4
     },
     {
-      "completion_length": 206.65625,
+      "completion_length": 251.0625,
       "epoch": 1.5714285714285714,
-      "grad_norm": 1.5178693532943726,
-      "kl": 0.03174284310080111,
+      "grad_norm": 1.0129271745681763,
+      "kl": 0.0313154561445117,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 8.552556872367859,
-      "reward_std": 7.061087839305401,
-      "rewards/concensus_correctness_reward_func": 5.843437597155571,
+      "reward": 8.019082754850388,
+      "reward_std": 8.092861026525497,
+      "rewards/concensus_correctness_reward_func": 5.346374973654747,
       "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.37958788871765137,
+      "rewards/question_recreation_reward_func": 0.4450203813612461,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.6107812561094761,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5245625153183937,
       "step": 6
     },
     {
-      "completion_length": 211.125,
+      "completion_length": 220.83333333333334,
       "epoch": 2.0,
-      "grad_norm": 0.6583097577095032,
-      "kl": 0.027615504804998636,
+      "grad_norm": 0.34419217705726624,
+      "kl": 0.037938895324865975,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 10.130588610967001,
-      "reward_std": 9.325920085112253,
-      "rewards/concensus_correctness_reward_func": 6.703166683514913,
-      "rewards/consensus_reward_func": 0.8333333333333334,
+      "reward": 6.547804554303487,
+      "reward_std": 6.387558157245318,
+      "rewards/concensus_correctness_reward_func": 3.3993333329757056,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.5063388552516699,
+      "rewards/final_correctness_reward_func": 1.25,
+      "rewards/question_recreation_reward_func": 0.5420126914978027,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6710833311080933,
+      "rewards/xmlcount_reward_func": 0.6064583162466685,
       "step": 8
     },
     {
-      "completion_length": 217.78125,
+      "completion_length": 246.09375,
       "epoch": 2.571428571428571,
-      "grad_norm": 1.0431419610977173,
-      "kl": 0.03605657909065485,
+      "grad_norm": 0.7747398614883423,
+      "kl": 0.03019706322811544,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 6.483449876308441,
-      "reward_std": 6.295221641659737,
-      "rewards/concensus_correctness_reward_func": 3.76581247523427,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 10.88028371334076,
+      "reward_std": 5.568570181727409,
+      "rewards/concensus_correctness_reward_func": 7.757687486708164,
+      "rewards/consensus_reward_func": 0.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.5397312082350254,
+      "rewards/final_correctness_reward_func": 1.0625,
+      "rewards/question_recreation_reward_func": 0.6195028889924288,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.552906259894371,
+      "rewards/xmlcount_reward_func": 0.503093745559454,
       "step": 10
     },
     {
-      "completion_length": 228.75,
+      "completion_length": 217.0,
       "epoch": 3.0,
-      "grad_norm": 0.6969460844993591,
-      "kl": 0.031089524428049724,
+      "grad_norm": 0.6290006637573242,
+      "kl": 0.031218677138288815,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 6.5001393755276995,
-      "reward_std": 6.519809847076734,
-      "rewards/concensus_correctness_reward_func": 4.256999999284744,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 10.209730307261148,
+      "reward_std": 8.558895786603292,
+      "rewards/concensus_correctness_reward_func": 7.271083349982898,
+      "rewards/consensus_reward_func": 0.9166666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5833333333333334,
-      "rewards/question_recreation_reward_func": 0.36276430636644363,
+      "rewards/final_correctness_reward_func": 0.9166666666666666,
+      "rewards/question_recreation_reward_func": 0.37406328568855923,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.713708351055781,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.7104166746139526,
       "step": 12
     },
     {
-      "completion_length": 198.9375,
+      "completion_length": 240.28125,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.8872530460357666,
-      "kl": 0.02684053615666926,
+      "grad_norm": 1.2460353374481201,
+      "kl": 0.0295397297013551,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 5.913686141371727,
-      "reward_std": 4.713027611374855,
-      "rewards/concensus_correctness_reward_func": 2.8759999722242355,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 8.406872689723969,
+      "reward_std": 8.903597861528397,
+      "rewards/concensus_correctness_reward_func": 5.327687561511993,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.5502487141638994,
+      "rewards/final_correctness_reward_func": 1.1875,
+      "rewards/question_recreation_reward_func": 0.5604351814836264,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.7530625071376562,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5031250072643161,
       "step": 14
     },
     {
-      "completion_length": 221.125,
+      "completion_length": 294.5416666666667,
       "epoch": 4.0,
-      "grad_norm": 0.5982879400253296,
-      "kl": 0.029973368781308334,
+      "grad_norm": 0.658294677734375,
+      "kl": 0.026256889570504427,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 5.973727345466614,
-      "reward_std": 3.197078049182892,
-      "rewards/concensus_correctness_reward_func": 3.357833360632261,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 7.691578308741252,
+      "reward_std": 8.699561337629953,
+      "rewards/concensus_correctness_reward_func": 5.291999963422616,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.4214770123362541,
+      "rewards/final_correctness_reward_func": 0.9166666666666666,
+      "rewards/question_recreation_reward_func": 0.2864949281016986,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.041666666666666664,
-      "rewards/xmlcount_reward_func": 0.5694166744748751,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5297499895095825,
       "step": 16
     },
     {
-      "completion_length": 243.5,
+      "completion_length": 263.9375,
       "epoch": 4.571428571428571,
-      "grad_norm": 1.070564866065979,
-      "kl": 0.031115309335291386,
+      "grad_norm": 0.7823837995529175,
+      "kl": 0.027016623644158244,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 6.433173283934593,
-      "reward_std": 6.740131661295891,
-      "rewards/concensus_correctness_reward_func": 3.5397499799728394,
+      "reward": 6.381502516567707,
+      "reward_std": 6.173025002703071,
+      "rewards/concensus_correctness_reward_func": 3.6688750088214874,
       "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1875,
-      "rewards/question_recreation_reward_func": 0.5655796639621258,
+      "rewards/final_correctness_reward_func": 1.0625,
+      "rewards/question_recreation_reward_func": 0.6055650487542152,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7028437554836273,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.5758124906569719,
       "step": 18
     },
     {
-      "completion_length": 209.5,
+      "completion_length": 247.5,
       "epoch": 5.0,
-      "grad_norm": 0.7200393676757812,
-      "kl": 0.03196889193107685,
+      "grad_norm": 0.3980932831764221,
+      "kl": 0.02494565040494005,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 8.892809788386026,
-      "reward_std": 8.186325470606485,
-      "rewards/concensus_correctness_reward_func": 5.795250087976456,
-      "rewards/consensus_reward_func": 0.8333333333333334,
+      "reward": 8.35054769118627,
+      "reward_std": 8.040125290552774,
+      "rewards/concensus_correctness_reward_func": 5.31850000222524,
+      "rewards/consensus_reward_func": 0.9166666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.44406017164389294,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.39696428676446277,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5701666797200838,
+      "rewards/xmlcount_reward_func": 0.718416690826416,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.633521071402356e-05,
-      "train_runtime": 297.393,
-      "train_samples_per_second": 1.076,
-      "train_steps_per_second": 0.067
+      "train_loss": 2.5608179566916078e-05,
+      "train_runtime": 324.6357,
+      "train_samples_per_second": 0.986,
+      "train_steps_per_second": 0.062
     }
   ],
   "logging_steps": 2,