End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a43e31a0353c6a26011bf956975e628e90fd893c3a429e5782503121755a3159
+oid sha256:6fa92fb55adb10a0678a07baa15852792562e0d1953b51fcff1ebf71046757a6
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.583472033729776e-05,
-    "train_runtime": 406.4207,
+    "train_loss": 6.76935035395824e-05,
+    "train_runtime": 380.3235,
     "train_samples": 24,
-    "train_samples_per_second": 1.181,
-    "train_steps_per_second": 0.074
+    "train_samples_per_second": 1.262,
+    "train_steps_per_second": 0.079
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.583472033729776e-05,
-    "train_runtime": 406.4207,
+    "train_loss": 6.76935035395824e-05,
+    "train_runtime": 380.3235,
     "train_samples": 24,
-    "train_samples_per_second": 1.181,
-    "train_steps_per_second": 0.074
+    "train_samples_per_second": 1.262,
+    "train_steps_per_second": 0.079
 }

trainer_state.json

@@ -9,298 +9,298 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 281.53125,
+      "completion_length": 333.78125,
       "epoch": 0.3333333333333333,
-      "grad_norm": 2.773542642593384,
-      "kl": 0.0580324565526098,
+      "grad_norm": 2.23221492767334,
+      "kl": 0.05727095284964889,
       "learning_rate": 4.985344892885899e-07,
       "loss": 0.0001,
-      "reward": 2.4544273018836975,
-      "reward_std": 2.3720036819577217,
-      "rewards/concensus_correctness_reward_func": 0.8223749995231628,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.9450934082269669,
+      "reward_std": 1.730120688676834,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.5142397312447429,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5917184855788946,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5553124938160181,
+      "rewards/xmlcount_reward_func": 0.5408749803900719,
       "step": 2
     },
     {
-      "completion_length": 271.28125,
+      "completion_length": 312.71875,
       "epoch": 0.6666666666666666,
-      "grad_norm": 2.0007011890411377,
-      "kl": 0.07696286379359663,
+      "grad_norm": 2.403878688812256,
+      "kl": 0.08554882137104869,
       "learning_rate": 4.869132927957006e-07,
       "loss": 0.0001,
-      "reward": 1.217789612710476,
-      "reward_std": 1.0034673381596804,
-      "rewards/concensus_correctness_reward_func": 0.0598750002682209,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 2.863950189203024,
+      "reward_std": 2.569031774997711,
+      "rewards/concensus_correctness_reward_func": 1.369999997317791,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.36272712517529726,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.44188757985830307,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6076875105500221,
+      "rewards/xmlcount_reward_func": 0.48956248909235,
       "step": 4
     },
     {
-      "completion_length": 303.25,
+      "completion_length": 343.625,
       "epoch": 1.0,
-      "grad_norm": 6.666475772857666,
-      "kl": 0.0992446473101154,
+      "grad_norm": 1.2604225873947144,
+      "kl": 0.028510307834949344,
       "learning_rate": 4.642142940418973e-07,
-      "loss": 0.0001,
-      "reward": 1.2962229549884796,
-      "reward_std": 1.3163017462939024,
-      "rewards/concensus_correctness_reward_func": 0.10431249998509884,
-      "rewards/consensus_reward_func": 0.25,
+      "loss": 0.0,
+      "reward": 1.9936382537707686,
+      "reward_std": 1.554558202624321,
+      "rewards/concensus_correctness_reward_func": 0.22356250323355198,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.42141042836010456,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.5116695226170123,
       "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3798749968409538,
+      "rewards/xmlcount_reward_func": 0.30528124468401074,
       "step": 6
     },
     {
-      "completion_length": 256.21875,
+      "completion_length": 269.96875,
       "epoch": 1.3333333333333333,
-      "grad_norm": 2.719036340713501,
-      "kl": 0.08407116535818204,
+      "grad_norm": 1.9944454431533813,
+      "kl": 0.07280526106478646,
       "learning_rate": 4.314988729807827e-07,
       "loss": 0.0001,
-      "reward": 1.9946057051420212,
-      "reward_std": 1.059820756316185,
-      "rewards/concensus_correctness_reward_func": 0.3334375023841858,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.4901013299822807,
+      "reward_std": 1.149802852421999,
+      "rewards/concensus_correctness_reward_func": 0.018812499940395355,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.4136994816362858,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.441507613286376,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.684968750923872,
+      "rewards/xmlcount_reward_func": 0.4672812442295253,
       "step": 8
     },
     {
-      "completion_length": 321.0,
+      "completion_length": 331.09375,
       "epoch": 1.6666666666666665,
-      "grad_norm": 2.027712821960449,
-      "kl": 0.10444008558988571,
+      "grad_norm": 30.684772491455078,
+      "kl": 0.07077017636038363,
       "learning_rate": 3.902967663405956e-07,
       "loss": 0.0001,
-      "reward": 2.8985730558633804,
-      "reward_std": 2.1391075775027275,
-      "rewards/concensus_correctness_reward_func": 0.9828124903142452,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 3.6839485419914126,
+      "reward_std": 3.374626036733389,
+      "rewards/concensus_correctness_reward_func": 2.042874988168478,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6192605346441269,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.40101105347275734,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6715000197291374,
+      "rewards/xmlcount_reward_func": 0.6150625091977417,
       "step": 10
     },
     {
-      "completion_length": 289.78125,
+      "completion_length": 288.0,
       "epoch": 2.0,
-      "grad_norm": 3.4199697971343994,
-      "kl": 0.06481281516607851,
+      "grad_norm": 2.637801170349121,
+      "kl": 0.06282075913622975,
       "learning_rate": 3.4253453883497864e-07,
       "loss": 0.0001,
-      "reward": 2.0985126718878746,
-      "reward_std": 1.588769018650055,
-      "rewards/concensus_correctness_reward_func": 0.3907500021159649,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.6097149848937988,
+      "reward_std": 1.3047048449516296,
+      "rewards/concensus_correctness_reward_func": 0.1197499968111515,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5003876239061356,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5750275067985058,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6448750030249357,
+      "rewards/xmlcount_reward_func": 0.5399375148117542,
       "step": 12
     },
     {
-      "completion_length": 322.65625,
+      "completion_length": 333.0625,
       "epoch": 2.3333333333333335,
-      "grad_norm": 1.4510133266448975,
-      "kl": 0.09526054270099849,
+      "grad_norm": 8.6305513381958,
+      "kl": 0.10169121820945293,
       "learning_rate": 2.9044549913819124e-07,
       "loss": 0.0001,
-      "reward": 1.6081246063113213,
-      "reward_std": 1.3514070957899094,
-      "rewards/concensus_correctness_reward_func": 0.26324999891221523,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 2.1766048222780228,
+      "reward_std": 1.1718497909605503,
+      "rewards/concensus_correctness_reward_func": 0.16387499682605267,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.3845308255404234,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.49301106855273247,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5228437595069408,
+      "rewards/xmlcount_reward_func": 0.5197187401354313,
       "step": 14
     },
     {
-      "completion_length": 300.875,
+      "completion_length": 300.0,
       "epoch": 2.6666666666666665,
-      "grad_norm": 32.19089126586914,
-      "kl": 0.08323089242912829,
+      "grad_norm": 1.2753549814224243,
+      "kl": 0.06518209201749414,
       "learning_rate": 2.3646527285364563e-07,
       "loss": 0.0001,
-      "reward": 2.5157083868980408,
-      "reward_std": 3.12582578510046,
-      "rewards/concensus_correctness_reward_func": 1.4086875282227993,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 4.655350029468536,
+      "reward_std": 5.421490907669067,
+      "rewards/concensus_correctness_reward_func": 3.292874988168478,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.39945833198726177,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.5220377082005143,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.45756249502301216,
+      "rewards/xmlcount_reward_func": 0.2154375035315752,
       "step": 16
     },
     {
-      "completion_length": 367.65625,
+      "completion_length": 256.96875,
       "epoch": 3.0,
-      "grad_norm": 2.201923131942749,
-      "kl": 0.040471521380823106,
+      "grad_norm": 2.370971441268921,
+      "kl": 0.07046509714564309,
       "learning_rate": 1.8311791536769483e-07,
-      "loss": 0.0,
-      "reward": 1.7202124670147896,
-      "reward_std": 1.1163596734404564,
-      "rewards/concensus_correctness_reward_func": 0.15918750315904617,
+      "loss": 0.0001,
+      "reward": 1.1566908285021782,
+      "reward_std": 1.1407469268888235,
+      "rewards/concensus_correctness_reward_func": 0.12006250023841858,
       "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.5503374617546797,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4378783367574215,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.43256248952820897,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.41124999476596713,
       "step": 18
     },
     {
-      "completion_length": 298.71875,
+      "completion_length": 306.6875,
       "epoch": 3.3333333333333335,
-      "grad_norm": 5.080014705657959,
-      "kl": 0.1327864183112979,
+      "grad_norm": 3.7701992988586426,
+      "kl": 0.07338486751541495,
       "learning_rate": 1.328978898250525e-07,
       "loss": 0.0001,
-      "reward": 2.0473050475120544,
-      "reward_std": 1.830180624499917,
-      "rewards/concensus_correctness_reward_func": 0.684749998152256,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 2.597634121775627,
+      "reward_std": 2.288734745234251,
+      "rewards/concensus_correctness_reward_func": 0.7414374984800816,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.38293003430590034,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.4301653988659382,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7296250090003014,
+      "rewards/xmlcount_reward_func": 0.4260312672704458,
       "step": 20
     },
     {
-      "completion_length": 321.9375,
+      "completion_length": 231.65625,
       "epoch": 3.6666666666666665,
-      "grad_norm": 1.9241300821304321,
-      "kl": 0.07197530311532319,
+      "grad_norm": 33.05220031738281,
+      "kl": 0.06382088427199051,
       "learning_rate": 8.81534288045431e-08,
       "loss": 0.0001,
-      "reward": 1.3912755213677883,
-      "reward_std": 0.9820459727197886,
-      "rewards/concensus_correctness_reward_func": 0.10918749868869781,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.6381819306407124,
+      "reward_std": 1.1167348898015916,
+      "rewards/concensus_correctness_reward_func": 0.17299999669194221,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.4397130124270916,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.46974439593032,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5142499897629023,
+      "rewards/xmlcount_reward_func": 0.557937502861023,
       "step": 22
     },
     {
-      "completion_length": 347.75,
+      "completion_length": 250.75,
       "epoch": 4.0,
-      "grad_norm": 2.118330240249634,
-      "kl": 0.051218974171206355,
+      "grad_norm": 2.149594306945801,
+      "kl": 0.065125553868711,
       "learning_rate": 5.097673357358906e-08,
       "loss": 0.0001,
-      "reward": 2.470015749335289,
-      "reward_std": 3.3017385974526405,
-      "rewards/concensus_correctness_reward_func": 0.9248749949038029,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 3.1555401384830475,
+      "reward_std": 2.317109949886799,
+      "rewards/concensus_correctness_reward_func": 1.4325000010430813,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5278282500803471,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.251687491312623,
+      "rewards/question_recreation_reward_func": 0.5613213405013084,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.7085937708616257,
       "step": 24
     },
     {
-      "completion_length": 399.5625,
+      "completion_length": 318.3125,
       "epoch": 4.333333333333333,
-      "grad_norm": 1.9072325229644775,
-      "kl": 0.042736936593428254,
+      "grad_norm": 2.0100913047790527,
+      "kl": 0.06343707226915285,
       "learning_rate": 2.3106145082260774e-08,
-      "loss": 0.0,
-      "reward": 2.27708288282156,
-      "reward_std": 2.7430423460900784,
-      "rewards/concensus_correctness_reward_func": 0.8621874898672104,
-      "rewards/consensus_reward_func": 0.25,
+      "loss": 0.0001,
+      "reward": 1.0433421432971954,
+      "reward_std": 1.362046491354704,
+      "rewards/concensus_correctness_reward_func": 0.07881249859929085,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5607079211622477,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5773108880966902,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2291874964721501,
+      "rewards/xmlcount_reward_func": 0.26221874030306935,
       "step": 26
     },
     {
-      "completion_length": 354.09375,
+      "completion_length": 325.21875,
       "epoch": 4.666666666666667,
-      "grad_norm": 2.1720738410949707,
-      "kl": 0.08552559395320714,
+      "grad_norm": 1.5065702199935913,
+      "kl": 0.04980072972830385,
       "learning_rate": 5.844861072478335e-09,
-      "loss": 0.0001,
-      "reward": 1.2497381269931793,
-      "reward_std": 1.0915438868105412,
-      "rewards/concensus_correctness_reward_func": 0.14025000110268593,
-      "rewards/consensus_reward_func": 0.0625,
+      "loss": 0.0,
+      "reward": 3.545183375477791,
+      "reward_std": 3.6524936025962234,
+      "rewards/concensus_correctness_reward_func": 1.9014999996870756,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.44089438393712044,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.3512457935139537,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.5279687470756471,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.41743750870227814,
       "step": 28
     },
     {
-      "completion_length": 294.125,
+      "completion_length": 294.4375,
       "epoch": 5.0,
-      "grad_norm": 1.8672877550125122,
-      "kl": 0.04691072192508727,
+      "grad_norm": 1.806432843208313,
+      "kl": 0.08484870253596455,
       "learning_rate": 0.0,
-      "loss": 0.0,
-      "reward": 1.9176766276359558,
-      "reward_std": 1.8577748499810696,
-      "rewards/concensus_correctness_reward_func": 0.338749997317791,
-      "rewards/consensus_reward_func": 0.3125,
+      "loss": 0.0001,
+      "reward": 2.9638523161411285,
+      "reward_std": 2.550200469791889,
+      "rewards/concensus_correctness_reward_func": 0.9272499978542328,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5273328814655542,
+      "rewards/final_correctness_reward_func": 0.6875,
+      "rewards/question_recreation_reward_func": 0.42335233092308044,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5515937563031912,
+      "rewards/xmlcount_reward_func": 0.6757499873638153,
       "step": 30
     },
     {
       "epoch": 5.0,
       "step": 30,
       "total_flos": 0.0,
-      "train_loss": 7.583472033729776e-05,
-      "train_runtime": 406.4207,
-      "train_samples_per_second": 1.181,
-      "train_steps_per_second": 0.074
+      "train_loss": 6.76935035395824e-05,
+      "train_runtime": 380.3235,
+      "train_samples_per_second": 1.262,
+      "train_steps_per_second": 0.079
     }
   ],
   "logging_steps": 2,