End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.0466744899749755e-06,
-    "train_runtime": 277.5004,
+    "train_loss": 5.343183875083923e-06,
+    "train_runtime": 240.7007,
     "train_samples": 160,
-    "train_samples_per_second": 1.153,
-    "train_steps_per_second": 0.072
+    "train_samples_per_second": 1.329,
+    "train_steps_per_second": 0.083
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2890edb052d5a7a513a1df475855d993c0f09d5f2c68d6a28535b2f996dbb4a5
+oid sha256:bd2a14a50d5fc22447a85d354b80f55f640b1b5b81d942e22a63975c12d6f530
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.0466744899749755e-06,
-    "train_runtime": 277.5004,
+    "train_loss": 5.343183875083923e-06,
+    "train_runtime": 240.7007,
     "train_samples": 160,
-    "train_samples_per_second": 1.153,
-    "train_steps_per_second": 0.072
+    "train_samples_per_second": 1.329,
+    "train_steps_per_second": 0.083
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 535.375,
+      "completion_length": 407.53125,
       "epoch": 0.1,
-      "grad_norm": 3.157778024673462,
+      "grad_norm": 3.85707950592041,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 1.4152686716988683,
-      "reward_std": 1.4165472311433405,
-      "rewards/concensus_correctness_reward_func": 0.2554999999701977,
+      "loss": -0.0,
+      "reward": 2.093384373583831,
+      "reward_std": 1.590828881773632,
+      "rewards/concensus_correctness_reward_func": 0.7905625030398369,
       "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.41379993385635316,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.43754065490793437,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3084687483496964,
+      "rewards/xmlcount_reward_func": 0.3652812608052045,
       "step": 2
     },
     {
-      "completion_length": 368.625,
+      "completion_length": 416.90625,
       "epoch": 0.2,
-      "grad_norm": 4.812714576721191,
-      "kl": 0.0009780465734365862,
+      "grad_norm": 4.87860107421875,
+      "kl": 0.001041749714204343,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.011606238782406,
-      "reward_std": 1.1807999651646242,
-      "rewards/concensus_correctness_reward_func": 0.21856249868869781,
+      "reward": 1.574540438130498,
+      "reward_std": 1.2005670535309036,
+      "rewards/concensus_correctness_reward_func": 0.2815625034272671,
       "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5831062341458164,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.49441539542749524,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6474375100806355,
+      "rewards/xmlcount_reward_func": 0.36106250330340117,
       "step": 4
     },
     {
-      "completion_length": 435.75,
+      "completion_length": 349.84375,
       "epoch": 0.3,
-      "grad_norm": 3.5027971267700195,
-      "kl": 0.001512384525995003,
+      "grad_norm": 4.286645889282227,
+      "kl": 0.0022525005297211464,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.2831329992040992,
-      "reward_std": 0.783065743278712,
-      "rewards/concensus_correctness_reward_func": 0.12831249833106995,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 3.9954563453793526,
+      "reward_std": 1.5508513655513525,
+      "rewards/concensus_correctness_reward_func": 2.106687491759658,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.36978927883319557,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5791749977506697,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4100312478840351,
+      "rewards/xmlcount_reward_func": 0.6845937455072999,
       "step": 6
     },
     {
-      "completion_length": 390.125,
+      "completion_length": 406.625,
       "epoch": 0.4,
-      "grad_norm": 4.555922985076904,
-      "kl": 0.0026833822048502043,
+      "grad_norm": 4.579648017883301,
+      "kl": 0.0032924842307693325,
       "learning_rate": 3.867370395306068e-07,
-      "loss": -0.0,
-      "reward": 2.4401909597218037,
-      "reward_std": 1.6595184421203157,
-      "rewards/concensus_correctness_reward_func": 0.7971875034272671,
-      "rewards/consensus_reward_func": 0.1875,
+      "loss": 0.0,
+      "reward": 3.1603123545646667,
+      "reward_std": 2.947513014310971,
+      "rewards/concensus_correctness_reward_func": 1.577250000089407,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5569721886422485,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5846873226109892,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.648531261831522,
+      "rewards/xmlcount_reward_func": 0.5608750153332949,
       "step": 8
     },
     {
-      "completion_length": 317.78125,
+      "completion_length": 271.90625,
       "epoch": 0.5,
-      "grad_norm": 5.466304779052734,
-      "kl": 0.0031669778036302887,
+      "grad_norm": 4.632985591888428,
+      "kl": 0.004466589482035488,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 2.522922746837139,
-      "reward_std": 1.4969825763255358,
-      "rewards/concensus_correctness_reward_func": 0.4166875034570694,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 2.7470552176237106,
+      "reward_std": 1.6685950586106628,
+      "rewards/concensus_correctness_reward_func": 0.923250000923872,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.6001102740410715,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.4922426194534637,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6936250068247318,
+      "rewards/xmlcount_reward_func": 0.70656250230968,
       "step": 10
     },
     {
-      "completion_length": 359.28125,
+      "completion_length": 304.0,
       "epoch": 0.6,
-      "grad_norm": 4.209219455718994,
-      "kl": 0.004104424617253244,
+      "grad_norm": 5.20498514175415,
+      "kl": 0.006072886142646894,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.6193452179431915,
-      "reward_std": 1.180488772901299,
-      "rewards/concensus_correctness_reward_func": 0.24149999767541885,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.5954198315739632,
+      "reward_std": 0.8973580221645534,
+      "rewards/concensus_correctness_reward_func": 0.166812501847744,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5353139387443662,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5176073284819722,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5300312489271164,
+      "rewards/xmlcount_reward_func": 0.7235000031068921,
       "step": 12
     },
     {
-      "completion_length": 470.375,
+      "completion_length": 397.1875,
       "epoch": 0.7,
-      "grad_norm": 4.932950019836426,
-      "kl": 0.004521393813774921,
+      "grad_norm": 5.128066539764404,
+      "kl": 0.005728195843403228,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.8465154506266117,
-      "reward_std": 1.2842277802410536,
-      "rewards/concensus_correctness_reward_func": 0.1862499974668026,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 2.0546868927776814,
+      "reward_std": 1.442885822383687,
+      "rewards/concensus_correctness_reward_func": 0.27974999975413084,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5906404345296323,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.6354681567754596,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5071250074543059,
+      "rewards/xmlcount_reward_func": 0.6394687667489052,
       "step": 14
     },
     {
-      "completion_length": 437.53125,
+      "completion_length": 292.0,
       "epoch": 0.8,
-      "grad_norm": 5.087905406951904,
-      "kl": 0.004113712049729656,
+      "grad_norm": 4.532220840454102,
+      "kl": 0.00832171599904541,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 2.9451967539498582,
-      "reward_std": 2.026288349297829,
-      "rewards/concensus_correctness_reward_func": 1.0097499955445528,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.6703360937535763,
+      "reward_std": 1.0772570687986445,
+      "rewards/concensus_correctness_reward_func": 0.13393750321120024,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.6075405180454254,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.578429848770611,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5779062495566905,
+      "rewards/xmlcount_reward_func": 0.582968738861382,
       "step": 16
     },
     {
-      "completion_length": 329.71875,
+      "completion_length": 328.40625,
       "epoch": 0.9,
-      "grad_norm": 4.528990268707275,
-      "kl": 0.004846014897339046,
+      "grad_norm": 4.359241008758545,
+      "kl": 0.008372082724235952,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.04850946739316,
-      "reward_std": 1.0897686868556775,
-      "rewards/concensus_correctness_reward_func": 0.2551874965429306,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 2.5970488153398037,
+      "reward_std": 1.7320504037779756,
+      "rewards/concensus_correctness_reward_func": 0.9057499952614307,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5326969629968517,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5966425262158737,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.6825000094249845,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.6571562602184713,
       "step": 18
     },
     {
-      "completion_length": 420.59375,
+      "completion_length": 248.34375,
       "epoch": 1.0,
-      "grad_norm": 4.428182125091553,
-      "kl": 0.004977483637048863,
+      "grad_norm": 6.086843013763428,
+      "kl": 0.010363147550378926,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 2.6623120829463005,
-      "reward_std": 1.6725687012076378,
-      "rewards/concensus_correctness_reward_func": 0.7178750019520521,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.7468589022755623,
+      "reward_std": 1.1298442513216287,
+      "rewards/concensus_correctness_reward_func": 0.08599999733269215,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.6135308686643839,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.6217026421800256,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6434062523767352,
+      "rewards/xmlcount_reward_func": 0.6016562599688768,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.0466744899749755e-06,
-      "train_runtime": 277.5004,
-      "train_samples_per_second": 1.153,
-      "train_steps_per_second": 0.072
+      "train_loss": 5.343183875083923e-06,
+      "train_runtime": 240.7007,
+      "train_samples_per_second": 1.329,
+      "train_steps_per_second": 0.083
     }
   ],
   "logging_steps": 2,