End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0026065433954499894,
-    "train_runtime": 212.6075,
+    "train_loss": 0.16417283475748262,
+    "train_runtime": 209.2973,
     "train_samples": 320,
-    "train_samples_per_second": 3.01,
-    "train_steps_per_second": 0.188
+    "train_samples_per_second": 3.058,
+    "train_steps_per_second": 0.191
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e9e355898edf3698d3600a592eb43a57f5b97a8feb3b2ddc76884c22cd406fa
+oid sha256:ffa3b7c2eb22954ed5bdfe4babb455302b1f9ad01a8f89be61226951f602a331
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0026065433954499894,
-    "train_runtime": 212.6075,
+    "train_loss": 0.16417283475748262,
+    "train_runtime": 209.2973,
     "train_samples": 320,
-    "train_samples_per_second": 3.01,
-    "train_steps_per_second": 0.188
+    "train_samples_per_second": 3.058,
+    "train_steps_per_second": 0.191
 }

trainer_state.json

@@ -10,393 +10,393 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 115.71875,
+      "completion_length": 115.53125,
       "epoch": 0.05,
-      "grad_norm": 28.141786575317383,
+      "grad_norm": 37.139556884765625,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 3.080722341313958,
-      "reward_std": 0.6955505819059908,
-      "rewards/concensus_correctness_reward_func": 0.9308124892413616,
-      "rewards/consensus_reward_func": 0.875,
+      "reward": 2.4475310780107975,
+      "reward_std": 1.3624604484939482,
+      "rewards/concensus_correctness_reward_func": 0.4649375006556511,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.42944102408364415,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.48984355479478836,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.109375,
-      "rewards/xmlcount_reward_func": 0.7360937483608723,
+      "rewards/xmlcount_reward_func": 0.7583750076591969,
       "step": 2
     },
     {
-      "completion_length": 115.15625,
+      "completion_length": 108.34375,
       "epoch": 0.1,
-      "grad_norm": 18.37566375732422,
-      "kl": 0.017843228910351172,
+      "grad_norm": 20.202695846557617,
+      "kl": 0.014847530546830967,
       "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 4.108856491744518,
-      "reward_std": 1.4645002749748528,
-      "rewards/concensus_correctness_reward_func": 1.100500002503395,
-      "rewards/consensus_reward_func": 1.1875,
+      "reward": 5.1692750453948975,
+      "reward_std": 1.3825315139256418,
+      "rewards/concensus_correctness_reward_func": 1.6117499954998493,
+      "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5959815399255604,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6315875351428986,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 0.9748750030994415,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 0.9728124998509884,
       "step": 4
     },
     {
-      "completion_length": 101.96875,
+      "completion_length": 107.15625,
       "epoch": 0.15,
-      "grad_norm": 24.032485961914062,
-      "kl": 0.07059068378293887,
+      "grad_norm": 21.550765991210938,
+      "kl": 0.08239698398392648,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0001,
-      "reward": 5.183237671852112,
-      "reward_std": 0.9490681896859314,
-      "rewards/concensus_correctness_reward_func": 1.5907499957829714,
-      "rewards/consensus_reward_func": 1.6875,
+      "reward": 4.982221730053425,
+      "reward_std": 0.6625452353619039,
+      "rewards/concensus_correctness_reward_func": 1.4067500066012144,
+      "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.45873774215579033,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5759404562413692,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.0712500140070915,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.0776562541723251,
       "step": 6
     },
     {
-      "completion_length": 106.53125,
+      "completion_length": 116.3125,
       "epoch": 0.2,
-      "grad_norm": 23.589221954345703,
-      "kl": 0.9075778928818181,
+      "grad_norm": 26.379663467407227,
+      "kl": 0.2936959662474692,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0009,
-      "reward": 4.851357385516167,
-      "reward_std": 1.071611185092479,
-      "rewards/concensus_correctness_reward_func": 1.4482499975711107,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0003,
+      "reward": 5.107673645019531,
+      "reward_std": 2.3407255806960166,
+      "rewards/concensus_correctness_reward_func": 1.9551249891519547,
+      "rewards/consensus_reward_func": 1.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5514824334532022,
+      "rewards/question_recreation_reward_func": 0.6457360591739416,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.0235000029206276,
+      "rewards/strict_format_reward_func": 0.203125,
+      "rewards/xmlcount_reward_func": 0.9911875016987324,
       "step": 8
     },
     {
-      "completion_length": 107.4375,
+      "completion_length": 102.28125,
       "epoch": 0.25,
-      "grad_norm": 24.931913375854492,
-      "kl": 1.187446946860291,
+      "grad_norm": 25.451066970825195,
+      "kl": 0.7054350507678464,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0012,
-      "reward": 4.945584326982498,
-      "reward_std": 0.8899367619305849,
-      "rewards/concensus_correctness_reward_func": 1.4574999995529652,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0007,
+      "reward": 6.026900440454483,
+      "reward_std": 0.6154376133927144,
+      "rewards/concensus_correctness_reward_func": 1.9788749888539314,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6286780787631869,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5206817177822813,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.0000312514603138,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.13671875,
       "step": 10
     },
     {
-      "completion_length": 108.625,
+      "completion_length": 109.875,
       "epoch": 0.3,
-      "grad_norm": 21.122905731201172,
-      "kl": 0.7102811370277777,
+      "grad_norm": 24.288022994995117,
+      "kl": 2.1607862473465502,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0007,
-      "reward": 5.369517683982849,
-      "reward_std": 1.5619308175519109,
-      "rewards/concensus_correctness_reward_func": 1.6689374819397926,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0022,
+      "reward": 4.333971664309502,
+      "reward_std": 1.4009621833683923,
+      "rewards/concensus_correctness_reward_func": 1.214937500655651,
+      "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.615455181337893,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6222216123715043,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.0695000030100346,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 0.9968124963343143,
       "step": 12
     },
     {
-      "completion_length": 109.15625,
+      "completion_length": 104.625,
       "epoch": 0.35,
-      "grad_norm": 59.90224075317383,
-      "kl": 5.442961755674332,
+      "grad_norm": 28.804235458374023,
+      "kl": 0.6891685710288584,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0054,
-      "reward": 6.124573543667793,
-      "reward_std": 1.8788381005870178,
-      "rewards/concensus_correctness_reward_func": 2.6781250201165676,
+      "loss": 0.0007,
+      "reward": 5.217689886689186,
+      "reward_std": 0.9694035681604873,
+      "rewards/concensus_correctness_reward_func": 1.6384375020861626,
       "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5945110125467181,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5610647844150662,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 0.992562510073185,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 1.0181875005364418,
       "step": 14
     },
     {
-      "completion_length": 106.625,
+      "completion_length": 103.21875,
       "epoch": 0.4,
-      "grad_norm": 21.15949249267578,
-      "kl": 0.5498712724074721,
+      "grad_norm": 35.775413513183594,
+      "kl": 0.8724039853550494,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0005,
-      "reward": 5.564565867185593,
-      "reward_std": 0.9688403338659555,
-      "rewards/concensus_correctness_reward_func": 1.6811875104904175,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 0.0009,
+      "reward": 5.001754555851221,
+      "reward_std": 0.8620858918002341,
+      "rewards/concensus_correctness_reward_func": 1.480874989181757,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6686283685266972,
+      "rewards/question_recreation_reward_func": 0.5975045184604824,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.1053750030696392,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 1.0483750030398369,
       "step": 16
     },
     {
-      "completion_length": 112.1875,
+      "completion_length": 116.625,
       "epoch": 0.45,
-      "grad_norm": 34.32904052734375,
-      "kl": 11.562028194312006,
+      "grad_norm": 21.354928970336914,
+      "kl": 0.34439645055681467,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0116,
-      "reward": 5.582300066947937,
-      "reward_std": 1.7333503647241741,
-      "rewards/concensus_correctness_reward_func": 2.07431248947978,
-      "rewards/consensus_reward_func": 1.4375,
+      "loss": 0.0003,
+      "reward": 5.742968708276749,
+      "reward_std": 0.8694476368837059,
+      "rewards/concensus_correctness_reward_func": 1.9676875174045563,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.7077374998480082,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.6388748935423791,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.0033750012516975,
+      "rewards/strict_format_reward_func": 0.203125,
+      "rewards/xmlcount_reward_func": 0.9332812521606684,
       "step": 18
     },
     {
-      "completion_length": 117.71875,
+      "completion_length": 103.84375,
       "epoch": 0.5,
-      "grad_norm": 26.058582305908203,
-      "kl": 1.126984592527151,
+      "grad_norm": 57.22490310668945,
+      "kl": 1.7470143185928464,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0011,
-      "reward": 4.181420147418976,
-      "reward_std": 1.3247168653761037,
-      "rewards/concensus_correctness_reward_func": 1.1436875015497208,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0017,
+      "reward": 6.413925901055336,
+      "reward_std": 1.2340805160347372,
+      "rewards/concensus_correctness_reward_func": 2.8225624971091747,
+      "rewards/consensus_reward_func": 1.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5951076466590166,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5284571875818074,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 0.9738749992102385,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.0472812503576279,
       "step": 20
     },
     {
-      "completion_length": 106.71875,
+      "completion_length": 104.8125,
       "epoch": 0.55,
-      "grad_norm": 31.642139434814453,
-      "kl": 0.7342154434882104,
+      "grad_norm": 24.951213836669922,
+      "kl": 0.47896549943834543,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0007,
-      "reward": 4.2624650448560715,
-      "reward_std": 1.736710302066058,
-      "rewards/concensus_correctness_reward_func": 1.1389375030994415,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0005,
+      "reward": 5.5750699788331985,
+      "reward_std": 0.750914608783205,
+      "rewards/concensus_correctness_reward_func": 1.6819374971091747,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6370588149875402,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.6984137212857604,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 1.0177187509834766,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.1165937520563602,
       "step": 22
     },
     {
-      "completion_length": 99.46875,
+      "completion_length": 112.3125,
       "epoch": 0.6,
-      "grad_norm": 65.90033721923828,
-      "kl": 5.04171756003052,
+      "grad_norm": 21.453693389892578,
+      "kl": 0.5181310146581382,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.005,
-      "reward": 4.612813338637352,
-      "reward_std": 1.5941019735182635,
-      "rewards/concensus_correctness_reward_func": 1.2605624943971634,
-      "rewards/consensus_reward_func": 1.4375,
+      "loss": 0.0005,
+      "reward": 5.20386229455471,
+      "reward_std": 0.9414823611150496,
+      "rewards/concensus_correctness_reward_func": 1.5962499864399433,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4941882867133245,
+      "rewards/question_recreation_reward_func": 0.5575497858226299,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1080624982714653,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 1.0031874999403954,
       "step": 24
     },
     {
-      "completion_length": 106.1875,
+      "completion_length": 105.96875,
       "epoch": 0.65,
-      "grad_norm": 24.204206466674805,
-      "kl": 1.1527493232861161,
+      "grad_norm": 27.866104125976562,
+      "kl": 3222.633953056298,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0012,
-      "reward": 4.555570490658283,
-      "reward_std": 1.3411388245076523,
-      "rewards/concensus_correctness_reward_func": 1.3198125027120113,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 3.2226,
+      "reward": 5.229758635163307,
+      "reward_std": 1.963500058511272,
+      "rewards/concensus_correctness_reward_func": 1.6254375018179417,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5504454995971173,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5854774066247046,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.0446874983608723,
+      "rewards/strict_format_reward_func": 0.28125,
+      "rewards/xmlcount_reward_func": 1.0500937551259995,
       "step": 26
     },
     {
-      "completion_length": 110.03125,
+      "completion_length": 106.5,
       "epoch": 0.7,
-      "grad_norm": 24.5333194732666,
-      "kl": 9.58880496234633,
+      "grad_norm": 34.11473083496094,
+      "kl": 21.088456489145756,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0096,
-      "reward": 4.167467080056667,
-      "reward_std": 1.495884626870975,
-      "rewards/concensus_correctness_reward_func": 1.2058124989271164,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 0.0211,
+      "reward": 4.6278437823057175,
+      "reward_std": 0.7609359241323546,
+      "rewards/concensus_correctness_reward_func": 1.32150000333786,
+      "rewards/consensus_reward_func": 1.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5166545738466084,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5827812571078539,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.140625,
-      "rewards/xmlcount_reward_func": 0.9293750040233135,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 1.0829375013709068,
       "step": 28
     },
     {
-      "completion_length": 105.4375,
+      "completion_length": 97.34375,
       "epoch": 0.75,
-      "grad_norm": 45.431358337402344,
-      "kl": 1.2492264057509601,
+      "grad_norm": 24.32623291015625,
+      "kl": 9.820234577637166,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0013,
-      "reward": 5.73547525331378,
-      "reward_std": 2.064389802981168,
-      "rewards/concensus_correctness_reward_func": 2.322999984025955,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0098,
+      "reward": 5.891083002090454,
+      "reward_std": 1.2470036953454837,
+      "rewards/concensus_correctness_reward_func": 2.527562513947487,
+      "rewards/consensus_reward_func": 1.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5407251240685582,
+      "rewards/question_recreation_reward_func": 0.5704892324283719,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 0.9342499934136868,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.0586562529206276,
       "step": 30
     },
     {
-      "completion_length": 97.625,
+      "completion_length": 95.84375,
       "epoch": 0.8,
-      "grad_norm": 22.755542755126953,
-      "kl": 0.5674767629243433,
+      "grad_norm": 32.80009078979492,
+      "kl": 1.8810399202629924,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0006,
-      "reward": 4.501953202299774,
-      "reward_std": 1.0095994778093882,
-      "rewards/concensus_correctness_reward_func": 1.274562492966652,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0018,
+      "reward": 4.773266464471817,
+      "reward_std": 0.8334280538319945,
+      "rewards/concensus_correctness_reward_func": 1.3728750012814999,
+      "rewards/consensus_reward_func": 1.4375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5317657189443707,
+      "rewards/question_recreation_reward_func": 0.5120790302753448,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.0081249997019768,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.1070624999701977,
       "step": 32
     },
     {
-      "completion_length": 111.75,
+      "completion_length": 113.5,
       "epoch": 0.85,
-      "grad_norm": 48.653533935546875,
-      "kl": 7.750168148428202,
+      "grad_norm": 35.31277084350586,
+      "kl": 13.102250803261995,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0077,
-      "reward": 4.9789127856493,
-      "reward_std": 1.1694186381064355,
-      "rewards/concensus_correctness_reward_func": 1.4983125068247318,
+      "loss": 0.0131,
+      "reward": 4.830469772219658,
+      "reward_std": 1.5596857847704086,
+      "rewards/concensus_correctness_reward_func": 1.4805624820291996,
       "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6224753325805068,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5856571896001697,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 1.014375001192093,
+      "rewards/strict_format_reward_func": 0.203125,
+      "rewards/xmlcount_reward_func": 0.9986250065267086,
       "step": 34
     },
     {
-      "completion_length": 110.03125,
+      "completion_length": 103.9375,
       "epoch": 0.9,
-      "grad_norm": 22.457836151123047,
-      "kl": 1.2734893589513376,
+      "grad_norm": 25.091636657714844,
+      "kl": 1.2431108206510544,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0013,
-      "reward": 4.401850417256355,
-      "reward_std": 1.3007537302328274,
-      "rewards/concensus_correctness_reward_func": 1.2692500054836273,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0012,
+      "reward": 5.303227871656418,
+      "reward_std": 1.3237640612060204,
+      "rewards/concensus_correctness_reward_func": 1.5557500012218952,
+      "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6151004622224718,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5443528575124219,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 0.9393750000745058,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.125,
       "step": 36
     },
     {
-      "completion_length": 109.84375,
+      "completion_length": 104.90625,
       "epoch": 0.95,
-      "grad_norm": 20.912979125976562,
-      "kl": 0.6942824618890882,
+      "grad_norm": 34.469093322753906,
+      "kl": 1.4800151162780821,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0007,
-      "reward": 4.380308553576469,
-      "reward_std": 1.1902506751939654,
-      "rewards/concensus_correctness_reward_func": 1.2075624912977219,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0015,
+      "reward": 4.909093216061592,
+      "reward_std": 1.483071091468446,
+      "rewards/concensus_correctness_reward_func": 1.5221874974668026,
+      "rewards/consensus_reward_func": 1.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6136210327968001,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.561686968896538,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.0435000024735928,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.028343752026558,
       "step": 38
     },
     {
-      "completion_length": 108.0,
+      "completion_length": 103.34375,
       "epoch": 1.0,
-      "grad_norm": 21.68805503845215,
-      "kl": 2.4967825568746775,
+      "grad_norm": 39.51275634765625,
+      "kl": 4.372888013720512,
       "learning_rate": 0.0,
-      "loss": 0.0025,
-      "reward": 5.049069061875343,
-      "reward_std": 1.0629464890807867,
-      "rewards/concensus_correctness_reward_func": 1.4604999981820583,
+      "loss": 0.0044,
+      "reward": 5.756643958389759,
+      "reward_std": 1.508879809640348,
+      "rewards/concensus_correctness_reward_func": 2.2557499706745148,
       "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5341940210200846,
+      "rewards/question_recreation_reward_func": 0.5359564470127225,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.0856250040233135,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 1.027437500655651,
       "step": 40
     },
     {
       "epoch": 1.0,
       "step": 40,
       "total_flos": 0.0,
-      "train_loss": 0.0026065433954499894,
-      "train_runtime": 212.6075,
-      "train_samples_per_second": 3.01,
-      "train_steps_per_second": 0.188
+      "train_loss": 0.16417283475748262,
+      "train_runtime": 209.2973,
+      "train_samples_per_second": 3.058,
+      "train_steps_per_second": 0.191
     }
   ],
   "logging_steps": 2,