End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.343183875083923e-06,
-    "train_runtime": 240.7007,
+    "train_loss": 3.829412162303925e-06,
+    "train_runtime": 263.1353,
     "train_samples": 160,
-    "train_samples_per_second": 1.329,
-    "train_steps_per_second": 0.083
+    "train_samples_per_second": 1.216,
+    "train_steps_per_second": 0.076
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd2a14a50d5fc22447a85d354b80f55f640b1b5b81d942e22a63975c12d6f530
+oid sha256:f15d528e2e36f6de62fc6fda08ea4bcac2e8a376fa243f59dc9e7dc03c94aba8
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.343183875083923e-06,
-    "train_runtime": 240.7007,
+    "train_loss": 3.829412162303925e-06,
+    "train_runtime": 263.1353,
     "train_samples": 160,
-    "train_samples_per_second": 1.329,
-    "train_steps_per_second": 0.083
+    "train_samples_per_second": 1.216,
+    "train_steps_per_second": 0.076
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 407.53125,
+      "completion_length": 442.875,
       "epoch": 0.1,
-      "grad_norm": 3.85707950592041,
+      "grad_norm": 4.908999919891357,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 2.093384373583831,
-      "reward_std": 1.590828881773632,
-      "rewards/concensus_correctness_reward_func": 0.7905625030398369,
-      "rewards/consensus_reward_func": 0.1875,
+      "loss": 0.0,
+      "reward": 2.1628924598917365,
+      "reward_std": 1.6292787790298462,
+      "rewards/concensus_correctness_reward_func": 0.909500002861023,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.43754065490793437,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5116424206644297,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3652812608052045,
+      "rewards/xmlcount_reward_func": 0.30424999934621155,
       "step": 2
     },
     {
-      "completion_length": 416.90625,
+      "completion_length": 410.21875,
       "epoch": 0.2,
-      "grad_norm": 4.87860107421875,
-      "kl": 0.001041749714204343,
+      "grad_norm": 3.8213980197906494,
+      "kl": 0.0011583357190829702,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.574540438130498,
-      "reward_std": 1.2005670535309036,
-      "rewards/concensus_correctness_reward_func": 0.2815625034272671,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.6593156047165394,
+      "reward_std": 0.7778630193788558,
+      "rewards/concensus_correctness_reward_func": 0.21643750183284283,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.49441539542749524,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.47603437514044344,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.36106250330340117,
+      "rewards/xmlcount_reward_func": 0.5918437484651804,
       "step": 4
     },
     {
-      "completion_length": 349.84375,
+      "completion_length": 324.03125,
       "epoch": 0.3,
-      "grad_norm": 4.286645889282227,
-      "kl": 0.0022525005297211464,
+      "grad_norm": 4.4785003662109375,
+      "kl": 0.002048558340902673,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 3.9954563453793526,
-      "reward_std": 1.5508513655513525,
-      "rewards/concensus_correctness_reward_func": 2.106687491759658,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 2.3681117631495,
+      "reward_std": 1.0142140060197562,
+      "rewards/concensus_correctness_reward_func": 0.3446874972432852,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5791749977506697,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.46542426804080606,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6845937455072999,
+      "rewards/xmlcount_reward_func": 0.8080000057816505,
       "step": 6
     },
     {
-      "completion_length": 406.625,
+      "completion_length": 454.15625,
       "epoch": 0.4,
-      "grad_norm": 4.579648017883301,
-      "kl": 0.0032924842307693325,
+      "grad_norm": 4.800704002380371,
+      "kl": 0.002417401239654282,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 3.1603123545646667,
-      "reward_std": 2.947513014310971,
-      "rewards/concensus_correctness_reward_func": 1.577250000089407,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.4630326330661774,
+      "reward_std": 1.500903086271137,
+      "rewards/concensus_correctness_reward_func": 0.2278125025331974,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5846873226109892,
+      "rewards/question_recreation_reward_func": 0.5519388569518924,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5608750153332949,
+      "rewards/xmlcount_reward_func": 0.30828125961124897,
       "step": 8
     },
     {
-      "completion_length": 271.90625,
+      "completion_length": 310.40625,
       "epoch": 0.5,
-      "grad_norm": 4.632985591888428,
-      "kl": 0.004466589482035488,
+      "grad_norm": 6.513106822967529,
+      "kl": 0.004747492930619046,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 2.7470552176237106,
-      "reward_std": 1.6685950586106628,
-      "rewards/concensus_correctness_reward_func": 0.923250000923872,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.8123386055231094,
+      "reward_std": 1.5764636304229498,
+      "rewards/concensus_correctness_reward_func": 0.7538125030696392,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.4922426194534637,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5232136324048042,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.70656250230968,
+      "rewards/xmlcount_reward_func": 0.7228125035762787,
       "step": 10
     },
     {
-      "completion_length": 304.0,
+      "completion_length": 343.5,
       "epoch": 0.6,
-      "grad_norm": 5.20498514175415,
-      "kl": 0.006072886142646894,
+      "grad_norm": 4.955071449279785,
+      "kl": 0.004099714526091702,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.5954198315739632,
-      "reward_std": 0.8973580221645534,
-      "rewards/concensus_correctness_reward_func": 0.166812501847744,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.8588946387171745,
+      "reward_std": 1.26679428643547,
+      "rewards/concensus_correctness_reward_func": 0.264937499538064,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5176073284819722,
+      "rewards/question_recreation_reward_func": 0.5469571151770651,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7235000031068921,
+      "rewards/xmlcount_reward_func": 0.7345000132918358,
       "step": 12
     },
     {
-      "completion_length": 397.1875,
+      "completion_length": 384.65625,
       "epoch": 0.7,
-      "grad_norm": 5.128066539764404,
-      "kl": 0.005728195843403228,
+      "grad_norm": 3.3973560333251953,
+      "kl": 0.0034129829655284993,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.0546868927776814,
-      "reward_std": 1.442885822383687,
-      "rewards/concensus_correctness_reward_func": 0.27974999975413084,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.059538347646594,
+      "reward_std": 1.3035997105762362,
+      "rewards/concensus_correctness_reward_func": 0.36906250193715096,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.6354681567754596,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5691633056849241,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6394687667489052,
+      "rewards/xmlcount_reward_func": 0.6213125018402934,
       "step": 14
     },
     {
-      "completion_length": 292.0,
+      "completion_length": 292.4375,
       "epoch": 0.8,
-      "grad_norm": 4.532220840454102,
-      "kl": 0.00832171599904541,
+      "grad_norm": 4.502716541290283,
+      "kl": 0.007692649000091478,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.6703360937535763,
-      "reward_std": 1.0772570687986445,
-      "rewards/concensus_correctness_reward_func": 0.13393750321120024,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 3.6721948944032192,
+      "reward_std": 1.5720796391251497,
+      "rewards/concensus_correctness_reward_func": 1.1785624660551548,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.578429848770611,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.6455074540572241,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.582968738861382,
+      "rewards/xmlcount_reward_func": 0.9106250144541264,
       "step": 16
     },
     {
-      "completion_length": 328.40625,
+      "completion_length": 332.15625,
       "epoch": 0.9,
-      "grad_norm": 4.359241008758545,
-      "kl": 0.008372082724235952,
+      "grad_norm": 3.9202282428741455,
+      "kl": 0.006438435477321036,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.5970488153398037,
-      "reward_std": 1.7320504037779756,
-      "rewards/concensus_correctness_reward_func": 0.9057499952614307,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.4832431823015213,
+      "reward_std": 1.7433103364892304,
+      "rewards/concensus_correctness_reward_func": 0.7693750001490116,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5966425262158737,
+      "rewards/question_recreation_reward_func": 0.6646182015538216,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6571562602184713,
+      "rewards/xmlcount_reward_func": 0.7992499936372042,
       "step": 18
     },
     {
-      "completion_length": 248.34375,
+      "completion_length": 398.65625,
       "epoch": 1.0,
-      "grad_norm": 6.086843013763428,
-      "kl": 0.010363147550378926,
+      "grad_norm": 5.829668998718262,
+      "kl": 0.00668711714388337,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.7468589022755623,
-      "reward_std": 1.1298442513216287,
-      "rewards/concensus_correctness_reward_func": 0.08599999733269215,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 2.2545786015689373,
+      "reward_std": 1.7215171623975039,
+      "rewards/concensus_correctness_reward_func": 0.7259999979287386,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.6217026421800256,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5022348794154823,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6016562599688768,
+      "rewards/xmlcount_reward_func": 0.5888437572866678,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 5.343183875083923e-06,
-      "train_runtime": 240.7007,
-      "train_samples_per_second": 1.329,
-      "train_steps_per_second": 0.083
+      "train_loss": 3.829412162303925e-06,
+      "train_runtime": 263.1353,
+      "train_samples_per_second": 1.216,
+      "train_steps_per_second": 0.076
     }
   ],
   "logging_steps": 2,