End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0004582042783113138,
-    "train_runtime": 200.2102,
+    "train_loss": 0.3126821821319027,
+    "train_runtime": 186.5224,
     "train_samples": 160,
-    "train_samples_per_second": 1.598,
-    "train_steps_per_second": 0.1
+    "train_samples_per_second": 1.716,
+    "train_steps_per_second": 0.107
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c025891245c908925cdb1bade098413bd430ac207e930146603b82e1762baf3
+oid sha256:c8a4a37ab54474338c92a54016fdd18df71d926a436e4be9a21931fe055a92b5
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0004582042783113138,
-    "train_runtime": 200.2102,
+    "train_loss": 0.3126821821319027,
+    "train_runtime": 186.5224,
     "train_samples": 160,
-    "train_samples_per_second": 1.598,
-    "train_steps_per_second": 0.1
+    "train_samples_per_second": 1.716,
+    "train_steps_per_second": 0.107
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 291.3125,
+      "completion_length": 258.125,
       "epoch": 0.1,
-      "grad_norm": 11.007444381713867,
+      "grad_norm": 12.1038236618042,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
-      "loss": -0.0,
-      "reward": 3.2677544988691807,
-      "reward_std": 1.7156066985917278,
-      "rewards/concensus_correctness_reward_func": 0.9710625112056732,
-      "rewards/consensus_reward_func": 0.9375,
+      "loss": 0.0,
+      "reward": 2.3500917481724173,
+      "reward_std": 0.9744344543432817,
+      "rewards/concensus_correctness_reward_func": 0.5234999991953373,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.4526607843581587,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5244667693041265,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.15625,
-      "rewards/xmlcount_reward_func": 0.6252812389284372,
+      "rewards/strict_format_reward_func": 0.171875,
+      "rewards/xmlcount_reward_func": 0.5677500029560179,
       "step": 2
     },
     {
-      "completion_length": 125.96875,
+      "completion_length": 157.28125,
       "epoch": 0.2,
-      "grad_norm": 10.013463973999023,
-      "kl": 0.006026249873684719,
+      "grad_norm": 15.164268493652344,
+      "kl": 0.016867930413582144,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 5.447439439594746,
-      "reward_std": 0.656533848377876,
-      "rewards/concensus_correctness_reward_func": 2.117875003721565,
-      "rewards/consensus_reward_func": 1.125,
+      "reward": 4.654461152851582,
+      "reward_std": 1.1400904605397955,
+      "rewards/concensus_correctness_reward_func": 1.1535625024698675,
+      "rewards/consensus_reward_func": 1.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6272832257673144,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6927111456170678,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.171031253412366,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.1363125015050173,
       "step": 4
     },
     {
-      "completion_length": 181.90625,
+      "completion_length": 170.4375,
       "epoch": 0.3,
-      "grad_norm": 8.630983352661133,
-      "kl": 0.008643639157526195,
+      "grad_norm": 11.745673179626465,
+      "kl": 0.31349003181094304,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0,
-      "reward": 4.567201271653175,
-      "reward_std": 0.9425070192664862,
-      "rewards/concensus_correctness_reward_func": 1.0073125064373016,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 0.0003,
+      "reward": 5.0158578380942345,
+      "reward_std": 2.04010990288225,
+      "rewards/concensus_correctness_reward_func": 1.4669375028461218,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.7649512263014913,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.7887953128665686,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.123062502592802,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.1507500000298023,
       "step": 6
     },
     {
-      "completion_length": 202.75,
+      "completion_length": 140.59375,
       "epoch": 0.4,
-      "grad_norm": 9.41815185546875,
-      "kl": 0.009758036263519898,
+      "grad_norm": 450.44647216796875,
+      "kl": 18.68448106572032,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0,
-      "reward": 4.846075922250748,
-      "reward_std": 1.3068306120985653,
-      "rewards/concensus_correctness_reward_func": 1.4096875004470348,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0187,
+      "reward": 4.864694945514202,
+      "reward_std": 1.9919630960794166,
+      "rewards/concensus_correctness_reward_func": 1.2613750007003546,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6658884854987264,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.6198824238963425,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 0.8329999931156635,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.1709375083446503,
       "step": 8
     },
     {
-      "completion_length": 124.78125,
+      "completion_length": 131.8125,
       "epoch": 0.5,
-      "grad_norm": 13.283008575439453,
-      "kl": 1.8802428454800975,
+      "grad_norm": 47933.35546875,
+      "kl": 3090.915692106355,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0019,
-      "reward": 4.81413272023201,
-      "reward_std": 1.5979239206353668,
-      "rewards/concensus_correctness_reward_func": 1.2898124940693378,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 3.0909,
+      "reward": 4.455410748720169,
+      "reward_std": 0.9344652489380678,
+      "rewards/concensus_correctness_reward_func": 1.055249996483326,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6121327836299315,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.7835044432431459,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.1934375017881393,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.1322812512516975,
       "step": 10
     },
     {
-      "completion_length": 171.34375,
+      "completion_length": 170.25,
       "epoch": 0.6,
-      "grad_norm": 13.46960163116455,
-      "kl": 0.1479067819309421,
+      "grad_norm": 50.33361053466797,
+      "kl": 5.447311662370339,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0001,
-      "reward": 3.920488730072975,
-      "reward_std": 1.0469800612336257,
-      "rewards/concensus_correctness_reward_func": 0.8751249937340617,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0054,
+      "reward": 4.70535284280777,
+      "reward_std": 1.5298584499396384,
+      "rewards/concensus_correctness_reward_func": 1.2123750075697899,
+      "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.626676200889051,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6975716073065996,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.1530625000596046,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.1860312521457672,
       "step": 12
     },
     {
-      "completion_length": 171.875,
+      "completion_length": 132.25,
       "epoch": 0.7,
-      "grad_norm": 10.207493782043457,
-      "kl": 0.3098248659953242,
+      "grad_norm": 59.70350646972656,
+      "kl": 6.450560495257378,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0003,
-      "reward": 5.522464923560619,
-      "reward_std": 0.9926261357031763,
-      "rewards/concensus_correctness_reward_func": 1.6238124892115593,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0065,
+      "reward": 4.6385799795389175,
+      "reward_std": 1.0262601334834471,
+      "rewards/concensus_correctness_reward_func": 1.1698749922215939,
+      "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6991524333134294,
+      "rewards/question_recreation_reward_func": 0.6355173867195845,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.1213750019669533,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.145687498152256,
       "step": 14
     },
     {
-      "completion_length": 125.53125,
+      "completion_length": 142.03125,
       "epoch": 0.8,
-      "grad_norm": 14.890645027160645,
-      "kl": 0.3561095618206309,
+      "grad_norm": 21.690887451171875,
+      "kl": 2.6143888300284743,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0004,
-      "reward": 5.000173605978489,
-      "reward_std": 0.9733745492994785,
-      "rewards/concensus_correctness_reward_func": 1.2542499955743551,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0026,
+      "reward": 3.90111280977726,
+      "reward_std": 1.5290619616862386,
+      "rewards/concensus_correctness_reward_func": 0.7911250032484531,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.632454858161509,
+      "rewards/question_recreation_reward_func": 0.6498003372689709,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.2228437513113022,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.1633124947547913,
       "step": 16
     },
     {
-      "completion_length": 132.0625,
+      "completion_length": 130.65625,
       "epoch": 0.9,
-      "grad_norm": 14.71757698059082,
-      "kl": 1.7551078895339742,
+      "grad_norm": 14.880861282348633,
+      "kl": 0.7605363634356763,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0018,
-      "reward": 4.814139850437641,
-      "reward_std": 1.3504715226590633,
-      "rewards/concensus_correctness_reward_func": 1.237874997779727,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0008,
+      "reward": 5.188799202442169,
+      "reward_std": 1.0886195020284504,
+      "rewards/concensus_correctness_reward_func": 1.276500005275011,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.7378586363047361,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.7756116222590208,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.2290312498807907,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.214812494814396,
       "step": 18
     },
     {
-      "completion_length": 156.4375,
+      "completion_length": 131.8125,
       "epoch": 1.0,
-      "grad_norm": 13.396187782287598,
-      "kl": 0.10536776797380298,
+      "grad_norm": 21.948577880859375,
+      "kl": 1.6171908722026274,
       "learning_rate": 0.0,
-      "loss": 0.0001,
-      "reward": 5.5395242385566235,
-      "reward_std": 1.3330394681543112,
-      "rewards/concensus_correctness_reward_func": 1.607875008136034,
-      "rewards/consensus_reward_func": 1.4375,
+      "loss": 0.0016,
+      "reward": 5.907395571470261,
+      "reward_std": 1.0099625813309103,
+      "rewards/concensus_correctness_reward_func": 1.6532500013709068,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.7148992889560759,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.7589581478387117,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.0604999959468842,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.2139374986290932,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.0004582042783113138,
-      "train_runtime": 200.2102,
-      "train_samples_per_second": 1.598,
-      "train_steps_per_second": 0.1
+      "train_loss": 0.3126821821319027,
+      "train_runtime": 186.5224,
+      "train_samples_per_second": 1.716,
+      "train_steps_per_second": 0.107
     }
   ],
   "logging_steps": 2,