diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,16933 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.7333333333333334,
+  "eval_steps": 500,
+  "global_step": 1300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 144.9375,
+      "epoch": 0.0013333333333333333,
+      "grad_norm": 1.6926657083386978,
+      "kl": 0.0,
+      "learning_rate": 9.993333333333333e-07,
+      "loss": 0.0,
+      "reward": 1.5378787517547607,
+      "reward_std": 0.2193162590265274,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.5691287517547607,
+      "step": 1
+    },
+    {
+      "completion_length": 127.859375,
+      "epoch": 0.0026666666666666666,
+      "grad_norm": 2.8095402365592594,
+      "kl": 0.000774383544921875,
+      "learning_rate": 9.986666666666667e-07,
+      "loss": 0.0,
+      "reward": 1.75,
+      "reward_std": 0.22358438372612,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.78125,
+      "step": 2
+    },
+    {
+      "completion_length": 127.0,
+      "epoch": 0.004,
+      "grad_norm": 1.8864667616235922,
+      "kl": 0.0007171630859375,
+      "learning_rate": 9.98e-07,
+      "loss": 0.0,
+      "reward": 1.5297505855560303,
+      "reward_std": 0.21532170474529266,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.545375645160675,
+      "step": 3
+    },
+    {
+      "completion_length": 131.015625,
+      "epoch": 0.005333333333333333,
+      "grad_norm": 2.131209720953126,
+      "kl": 0.0009918212890625,
+      "learning_rate": 9.973333333333332e-07,
+      "loss": 0.0,
+      "reward": 1.5572680234909058,
+      "reward_std": 0.10823628306388855,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.5728930234909058,
+      "step": 4
+    },
+    {
+      "completion_length": 121.859375,
+      "epoch": 0.006666666666666667,
+      "grad_norm": 2.1951797033943365,
+      "kl": 0.00145721435546875,
+      "learning_rate": 9.966666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.7590301036834717,
+      "reward_std": 0.16469071805477142,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7902800440788269,
+      "step": 5
+    },
+    {
+      "completion_length": 126.59375,
+      "epoch": 0.008,
+      "grad_norm": 4.966020297317633,
+      "kl": 0.003204345703125,
+      "learning_rate": 9.959999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.4139273166656494,
+      "reward_std": 0.12990710139274597,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.42955222725868225,
+      "step": 6
+    },
+    {
+      "completion_length": 137.4375,
+      "epoch": 0.009333333333333334,
+      "grad_norm": 2.794156803115764,
+      "kl": 0.0037078857421875,
+      "learning_rate": 9.953333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.4338620901107788,
+      "reward_std": 0.14533525705337524,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.4651120901107788,
+      "step": 7
+    },
+    {
+      "completion_length": 122.3125,
+      "epoch": 0.010666666666666666,
+      "grad_norm": 2.0072027045531633,
+      "kl": 0.00616455078125,
+      "learning_rate": 9.946666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.6881155967712402,
+      "reward_std": 0.13905756175518036,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6881154775619507,
+      "step": 8
+    },
+    {
+      "completion_length": 119.65625,
+      "epoch": 0.012,
+      "grad_norm": 2.4706927404169527,
+      "kl": 0.0079345703125,
+      "learning_rate": 9.94e-07,
+      "loss": 0.0003,
+      "reward": 1.6614583730697632,
+      "reward_std": 0.22499720752239227,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.6927083730697632,
+      "step": 9
+    },
+    {
+      "completion_length": 156.015625,
+      "epoch": 0.013333333333333334,
+      "grad_norm": 1.9463275132424676,
+      "kl": 0.0068359375,
+      "learning_rate": 9.933333333333333e-07,
+      "loss": 0.0003,
+      "reward": 1.5248935222625732,
+      "reward_std": 0.30331259965896606,
+      "rewards/format_reward": 0.890625,
+      "rewards/iou_reward": 0.6342684030532837,
+      "step": 10
+    },
+    {
+      "completion_length": 153.9375,
+      "epoch": 0.014666666666666666,
+      "grad_norm": 2.5387694182441973,
+      "kl": 0.007537841796875,
+      "learning_rate": 9.926666666666666e-07,
+      "loss": 0.0003,
+      "reward": 1.3840045928955078,
+      "reward_std": 0.32267558574676514,
+      "rewards/format_reward": 0.90625,
+      "rewards/iou_reward": 0.47775471210479736,
+      "step": 11
+    },
+    {
+      "completion_length": 142.734375,
+      "epoch": 0.016,
+      "grad_norm": 2.2214248703303214,
+      "kl": 0.010498046875,
+      "learning_rate": 9.92e-07,
+      "loss": 0.0004,
+      "reward": 1.5669978857040405,
+      "reward_std": 0.3243110775947571,
+      "rewards/format_reward": 0.90625,
+      "rewards/iou_reward": 0.6607478857040405,
+      "step": 12
+    },
+    {
+      "completion_length": 143.15625,
+      "epoch": 0.017333333333333333,
+      "grad_norm": 4.69508147461904,
+      "kl": 0.01202392578125,
+      "learning_rate": 9.913333333333333e-07,
+      "loss": 0.0005,
+      "reward": 1.4508929252624512,
+      "reward_std": 0.3787879943847656,
+      "rewards/format_reward": 0.890625,
+      "rewards/iou_reward": 0.5602678656578064,
+      "step": 13
+    },
+    {
+      "completion_length": 133.078125,
+      "epoch": 0.018666666666666668,
+      "grad_norm": 1.7118007436129112,
+      "kl": 0.00775146484375,
+      "learning_rate": 9.906666666666667e-07,
+      "loss": 0.0003,
+      "reward": 1.7263106107711792,
+      "reward_std": 0.20339563488960266,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.804435670375824,
+      "step": 14
+    },
+    {
+      "completion_length": 127.0,
+      "epoch": 0.02,
+      "grad_norm": 2.8076142015840273,
+      "kl": 0.030029296875,
+      "learning_rate": 9.9e-07,
+      "loss": 0.0012,
+      "reward": 1.6635687351226807,
+      "reward_std": 0.14473286271095276,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.6948187351226807,
+      "step": 15
+    },
+    {
+      "completion_length": 134.1875,
+      "epoch": 0.021333333333333333,
+      "grad_norm": 1.7842971339170917,
+      "kl": 0.01226806640625,
+      "learning_rate": 9.893333333333332e-07,
+      "loss": 0.0005,
+      "reward": 1.7224702835083008,
+      "reward_std": 0.13813602924346924,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7537202835083008,
+      "step": 16
+    },
+    {
+      "completion_length": 125.125,
+      "epoch": 0.02266666666666667,
+      "grad_norm": 2.420181980499645,
+      "kl": 0.010498046875,
+      "learning_rate": 9.886666666666665e-07,
+      "loss": 0.0004,
+      "reward": 1.8121962547302246,
+      "reward_std": 0.1385866105556488,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8121961951255798,
+      "step": 17
+    },
+    {
+      "completion_length": 120.75,
+      "epoch": 0.024,
+      "grad_norm": 1.876380378629883,
+      "kl": 0.01422119140625,
+      "learning_rate": 9.88e-07,
+      "loss": 0.0006,
+      "reward": 1.6469494104385376,
+      "reward_std": 0.11407830566167831,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6625744104385376,
+      "step": 18
+    },
+    {
+      "completion_length": 147.109375,
+      "epoch": 0.025333333333333333,
+      "grad_norm": 2.171878770555434,
+      "kl": 0.01422119140625,
+      "learning_rate": 9.873333333333333e-07,
+      "loss": 0.0006,
+      "reward": 1.519289493560791,
+      "reward_std": 0.26987820863723755,
+      "rewards/format_reward": 0.875,
+      "rewards/iou_reward": 0.6442894339561462,
+      "step": 19
+    },
+    {
+      "completion_length": 123.859375,
+      "epoch": 0.02666666666666667,
+      "grad_norm": 2.057654616341878,
+      "kl": 0.0185546875,
+      "learning_rate": 9.866666666666666e-07,
+      "loss": 0.0007,
+      "reward": 1.736718773841858,
+      "reward_std": 0.1819593906402588,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7835937738418579,
+      "step": 20
+    },
+    {
+      "completion_length": 116.265625,
+      "epoch": 0.028,
+      "grad_norm": 3.4014885016099923,
+      "kl": 0.043212890625,
+      "learning_rate": 9.86e-07,
+      "loss": 0.0017,
+      "reward": 1.5605902671813965,
+      "reward_std": 0.16192789375782013,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.5605902671813965,
+      "step": 21
+    },
+    {
+      "completion_length": 115.203125,
+      "epoch": 0.029333333333333333,
+      "grad_norm": 1.814871472771573,
+      "kl": 0.021484375,
+      "learning_rate": 9.853333333333333e-07,
+      "loss": 0.0009,
+      "reward": 1.8528646230697632,
+      "reward_std": 0.07076727598905563,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8684895634651184,
+      "step": 22
+    },
+    {
+      "completion_length": 130.109375,
+      "epoch": 0.030666666666666665,
+      "grad_norm": 2.9193535821665715,
+      "kl": 0.020263671875,
+      "learning_rate": 9.846666666666667e-07,
+      "loss": 0.0008,
+      "reward": 1.6060268878936768,
+      "reward_std": 0.2094947099685669,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.6372767686843872,
+      "step": 23
+    },
+    {
+      "completion_length": 123.4375,
+      "epoch": 0.032,
+      "grad_norm": 1.685992300759521,
+      "kl": 0.01177978515625,
+      "learning_rate": 9.84e-07,
+      "loss": 0.0005,
+      "reward": 1.7213541269302368,
+      "reward_std": 0.2131677269935608,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7369792461395264,
+      "step": 24
+    },
+    {
+      "completion_length": 107.96875,
+      "epoch": 0.03333333333333333,
+      "grad_norm": 2.2308406110590178,
+      "kl": 0.01531982421875,
+      "learning_rate": 9.833333333333332e-07,
+      "loss": 0.0006,
+      "reward": 1.8104166984558105,
+      "reward_std": 0.1522313803434372,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8104166984558105,
+      "step": 25
+    },
+    {
+      "completion_length": 123.34375,
+      "epoch": 0.034666666666666665,
+      "grad_norm": 2.138477738039809,
+      "kl": 0.0133056640625,
+      "learning_rate": 9.826666666666667e-07,
+      "loss": 0.0005,
+      "reward": 1.677343726158142,
+      "reward_std": 0.23269201815128326,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7398437261581421,
+      "step": 26
+    },
+    {
+      "completion_length": 131.734375,
+      "epoch": 0.036,
+      "grad_norm": 2.8538741375174963,
+      "kl": 0.0125732421875,
+      "learning_rate": 9.819999999999999e-07,
+      "loss": 0.0005,
+      "reward": 1.718154788017273,
+      "reward_std": 0.26465046405792236,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.780654788017273,
+      "step": 27
+    },
+    {
+      "completion_length": 127.203125,
+      "epoch": 0.037333333333333336,
+      "grad_norm": 2.5269134101077753,
+      "kl": 0.0272216796875,
+      "learning_rate": 9.813333333333332e-07,
+      "loss": 0.0011,
+      "reward": 1.6456845998764038,
+      "reward_std": 0.22269532084465027,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.6769344806671143,
+      "step": 28
+    },
+    {
+      "completion_length": 111.3125,
+      "epoch": 0.03866666666666667,
+      "grad_norm": 2.3797973687686187,
+      "kl": 0.0118408203125,
+      "learning_rate": 9.806666666666666e-07,
+      "loss": 0.0005,
+      "reward": 1.6520088911056519,
+      "reward_std": 0.22475482523441315,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6520088911056519,
+      "step": 29
+    },
+    {
+      "completion_length": 114.484375,
+      "epoch": 0.04,
+      "grad_norm": 2.5556925110068707,
+      "kl": 0.021728515625,
+      "learning_rate": 9.8e-07,
+      "loss": 0.0009,
+      "reward": 1.6233259439468384,
+      "reward_std": 0.1949889063835144,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.7014509439468384,
+      "step": 30
+    },
+    {
+      "completion_length": 106.28125,
+      "epoch": 0.04133333333333333,
+      "grad_norm": 1.928855901721541,
+      "kl": 0.026611328125,
+      "learning_rate": 9.793333333333333e-07,
+      "loss": 0.0011,
+      "reward": 1.757552146911621,
+      "reward_std": 0.06719288229942322,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7575521469116211,
+      "step": 31
+    },
+    {
+      "completion_length": 145.046875,
+      "epoch": 0.042666666666666665,
+      "grad_norm": 1.25537225282921,
+      "kl": 0.0203857421875,
+      "learning_rate": 9.786666666666666e-07,
+      "loss": 0.0008,
+      "reward": 1.5704984664916992,
+      "reward_std": 0.07414072751998901,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.648623526096344,
+      "step": 32
+    },
+    {
+      "completion_length": 104.21875,
+      "epoch": 0.044,
+      "grad_norm": 3.441957128954691,
+      "kl": 0.0230712890625,
+      "learning_rate": 9.78e-07,
+      "loss": 0.0009,
+      "reward": 1.771875023841858,
+      "reward_std": 0.14871905744075775,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7874999642372131,
+      "step": 33
+    },
+    {
+      "completion_length": 114.5625,
+      "epoch": 0.04533333333333334,
+      "grad_norm": 2.882288544318554,
+      "kl": 0.024169921875,
+      "learning_rate": 9.773333333333333e-07,
+      "loss": 0.001,
+      "reward": 1.714620590209961,
+      "reward_std": 0.13999660313129425,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7146204710006714,
+      "step": 34
+    },
+    {
+      "completion_length": 107.046875,
+      "epoch": 0.04666666666666667,
+      "grad_norm": 2.440240241524622,
+      "kl": 0.0225830078125,
+      "learning_rate": 9.766666666666667e-07,
+      "loss": 0.0009,
+      "reward": 1.7865885496139526,
+      "reward_std": 0.1592077612876892,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7865885496139526,
+      "step": 35
+    },
+    {
+      "completion_length": 114.0,
+      "epoch": 0.048,
+      "grad_norm": 1.5562261453817199,
+      "kl": 0.02392578125,
+      "learning_rate": 9.759999999999998e-07,
+      "loss": 0.001,
+      "reward": 1.7479166984558105,
+      "reward_std": 0.08517226576805115,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7635416984558105,
+      "step": 36
+    },
+    {
+      "completion_length": 112.734375,
+      "epoch": 0.04933333333333333,
+      "grad_norm": 6.432994441078256,
+      "kl": 0.01904296875,
+      "learning_rate": 9.753333333333334e-07,
+      "loss": 0.0008,
+      "reward": 1.7322916984558105,
+      "reward_std": 0.08135949820280075,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7479166984558105,
+      "step": 37
+    },
+    {
+      "completion_length": 112.875,
+      "epoch": 0.050666666666666665,
+      "grad_norm": 2.429619037161033,
+      "kl": 0.0289306640625,
+      "learning_rate": 9.746666666666666e-07,
+      "loss": 0.0012,
+      "reward": 1.7861979007720947,
+      "reward_std": 0.09846894443035126,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8018229007720947,
+      "step": 38
+    },
+    {
+      "completion_length": 110.984375,
+      "epoch": 0.052,
+      "grad_norm": 3.8511846177299747,
+      "kl": 0.0311279296875,
+      "learning_rate": 9.74e-07,
+      "loss": 0.0012,
+      "reward": 1.5953993797302246,
+      "reward_std": 0.17708733677864075,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6110242605209351,
+      "step": 39
+    },
+    {
+      "completion_length": 112.34375,
+      "epoch": 0.05333333333333334,
+      "grad_norm": 3.0490601997818425,
+      "kl": 0.0260009765625,
+      "learning_rate": 9.733333333333333e-07,
+      "loss": 0.001,
+      "reward": 1.606436014175415,
+      "reward_std": 0.22361630201339722,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.637686014175415,
+      "step": 40
+    },
+    {
+      "completion_length": 105.0,
+      "epoch": 0.05466666666666667,
+      "grad_norm": 17.495665104888612,
+      "kl": 0.022216796875,
+      "learning_rate": 9.726666666666666e-07,
+      "loss": 0.0009,
+      "reward": 1.808370590209961,
+      "reward_std": 0.042990148067474365,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8083705306053162,
+      "step": 41
+    },
+    {
+      "completion_length": 119.484375,
+      "epoch": 0.056,
+      "grad_norm": 1.9623980653397355,
+      "kl": 0.017578125,
+      "learning_rate": 9.72e-07,
+      "loss": 0.0007,
+      "reward": 1.7518229484558105,
+      "reward_std": 0.21709519624710083,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7674478888511658,
+      "step": 42
+    },
+    {
+      "completion_length": 129.203125,
+      "epoch": 0.05733333333333333,
+      "grad_norm": 5.265623669749024,
+      "kl": 0.023681640625,
+      "learning_rate": 9.713333333333333e-07,
+      "loss": 0.0009,
+      "reward": 1.8046875,
+      "reward_std": 0.09939013421535492,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8359375,
+      "step": 43
+    },
+    {
+      "completion_length": 118.46875,
+      "epoch": 0.058666666666666666,
+      "grad_norm": 2.944020080858007,
+      "kl": 0.0302734375,
+      "learning_rate": 9.706666666666667e-07,
+      "loss": 0.0012,
+      "reward": 1.7065105438232422,
+      "reward_std": 0.19013354182243347,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7221354246139526,
+      "step": 44
+    },
+    {
+      "completion_length": 121.59375,
+      "epoch": 0.06,
+      "grad_norm": 1.6274208841667344,
+      "kl": 0.0277099609375,
+      "learning_rate": 9.7e-07,
+      "loss": 0.0011,
+      "reward": 1.7747396230697632,
+      "reward_std": 0.10720261931419373,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7747395634651184,
+      "step": 45
+    },
+    {
+      "completion_length": 115.40625,
+      "epoch": 0.06133333333333333,
+      "grad_norm": 7.138158590256093,
+      "kl": 0.032470703125,
+      "learning_rate": 9.693333333333334e-07,
+      "loss": 0.0013,
+      "reward": 1.6065104007720947,
+      "reward_std": 0.21420659124851227,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6065104603767395,
+      "step": 46
+    },
+    {
+      "completion_length": 106.203125,
+      "epoch": 0.06266666666666666,
+      "grad_norm": 3.6148732044321403,
+      "kl": 0.0218505859375,
+      "learning_rate": 9.686666666666667e-07,
+      "loss": 0.0009,
+      "reward": 1.826562523841858,
+      "reward_std": 0.15275555849075317,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8265625238418579,
+      "step": 47
+    },
+    {
+      "completion_length": 120.984375,
+      "epoch": 0.064,
+      "grad_norm": 1.6475693391869308,
+      "kl": 0.025390625,
+      "learning_rate": 9.679999999999999e-07,
+      "loss": 0.001,
+      "reward": 1.7031621932983398,
+      "reward_std": 0.05195062234997749,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7031621932983398,
+      "step": 48
+    },
+    {
+      "completion_length": 114.8125,
+      "epoch": 0.06533333333333333,
+      "grad_norm": 2.651182340364488,
+      "kl": 0.0274658203125,
+      "learning_rate": 9.673333333333332e-07,
+      "loss": 0.0011,
+      "reward": 1.6864583492279053,
+      "reward_std": 0.22801385819911957,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.71770840883255,
+      "step": 49
+    },
+    {
+      "completion_length": 125.140625,
+      "epoch": 0.06666666666666667,
+      "grad_norm": 2.028045444377758,
+      "kl": 0.025146484375,
+      "learning_rate": 9.666666666666666e-07,
+      "loss": 0.001,
+      "reward": 1.659895896911621,
+      "reward_std": 0.21110254526138306,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6755207777023315,
+      "step": 50
+    },
+    {
+      "completion_length": 106.890625,
+      "epoch": 0.068,
+      "grad_norm": 5.616074509964338,
+      "kl": 0.0233154296875,
+      "learning_rate": 9.66e-07,
+      "loss": 0.0009,
+      "reward": 1.8171875476837158,
+      "reward_std": 0.10045116394758224,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.817187488079071,
+      "step": 51
+    },
+    {
+      "completion_length": 120.46875,
+      "epoch": 0.06933333333333333,
+      "grad_norm": 4.659636095478185,
+      "kl": 0.0257568359375,
+      "learning_rate": 9.653333333333333e-07,
+      "loss": 0.001,
+      "reward": 1.8010417222976685,
+      "reward_std": 0.10741767287254333,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.8635416626930237,
+      "step": 52
+    },
+    {
+      "completion_length": 122.46875,
+      "epoch": 0.07066666666666667,
+      "grad_norm": 2.7919637542047635,
+      "kl": 0.0216064453125,
+      "learning_rate": 9.646666666666666e-07,
+      "loss": 0.0009,
+      "reward": 1.7429687976837158,
+      "reward_std": 0.1388859897851944,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7429687976837158,
+      "step": 53
+    },
+    {
+      "completion_length": 108.53125,
+      "epoch": 0.072,
+      "grad_norm": 4.498643386965167,
+      "kl": 0.03759765625,
+      "learning_rate": 9.64e-07,
+      "loss": 0.0015,
+      "reward": 1.844847559928894,
+      "reward_std": 0.049905337393283844,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8448475003242493,
+      "step": 54
+    },
+    {
+      "completion_length": 126.296875,
+      "epoch": 0.07333333333333333,
+      "grad_norm": 3.1944089960891895,
+      "kl": 0.049072265625,
+      "learning_rate": 9.633333333333334e-07,
+      "loss": 0.002,
+      "reward": 1.7768229246139526,
+      "reward_std": 0.14310096204280853,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7768229246139526,
+      "step": 55
+    },
+    {
+      "completion_length": 123.9375,
+      "epoch": 0.07466666666666667,
+      "grad_norm": 2.428686804284596,
+      "kl": 0.025390625,
+      "learning_rate": 9.626666666666667e-07,
+      "loss": 0.001,
+      "reward": 1.703385353088379,
+      "reward_std": 0.21391814947128296,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7346354126930237,
+      "step": 56
+    },
+    {
+      "completion_length": 119.515625,
+      "epoch": 0.076,
+      "grad_norm": 2.913544558057168,
+      "kl": 0.0194091796875,
+      "learning_rate": 9.619999999999999e-07,
+      "loss": 0.0008,
+      "reward": 1.725000023841858,
+      "reward_std": 0.19347397983074188,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7562500238418579,
+      "step": 57
+    },
+    {
+      "completion_length": 120.390625,
+      "epoch": 0.07733333333333334,
+      "grad_norm": 1.239932514282945,
+      "kl": 0.0264892578125,
+      "learning_rate": 9.613333333333334e-07,
+      "loss": 0.0011,
+      "reward": 1.8098958730697632,
+      "reward_std": 0.05355679243803024,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8411458730697632,
+      "step": 58
+    },
+    {
+      "completion_length": 118.171875,
+      "epoch": 0.07866666666666666,
+      "grad_norm": 2.6957590815796832,
+      "kl": 0.025634765625,
+      "learning_rate": 9.606666666666666e-07,
+      "loss": 0.001,
+      "reward": 1.8331100940704346,
+      "reward_std": 0.09402933716773987,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8331100940704346,
+      "step": 59
+    },
+    {
+      "completion_length": 126.28125,
+      "epoch": 0.08,
+      "grad_norm": 1.9369473285686254,
+      "kl": 0.0164794921875,
+      "learning_rate": 9.6e-07,
+      "loss": 0.0007,
+      "reward": 1.7101563215255737,
+      "reward_std": 0.161530002951622,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7101562023162842,
+      "step": 60
+    },
+    {
+      "completion_length": 126.4375,
+      "epoch": 0.08133333333333333,
+      "grad_norm": 2.1237020134026543,
+      "kl": 0.02001953125,
+      "learning_rate": 9.593333333333333e-07,
+      "loss": 0.0008,
+      "reward": 1.7572916746139526,
+      "reward_std": 0.15625,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7572916746139526,
+      "step": 61
+    },
+    {
+      "completion_length": 123.8125,
+      "epoch": 0.08266666666666667,
+      "grad_norm": 1.8055274081257562,
+      "kl": 0.01953125,
+      "learning_rate": 9.586666666666666e-07,
+      "loss": 0.0008,
+      "reward": 1.65234375,
+      "reward_std": 0.14215977489948273,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.65234375,
+      "step": 62
+    },
+    {
+      "completion_length": 116.21875,
+      "epoch": 0.084,
+      "grad_norm": 3.1527578193259136,
+      "kl": 0.0159912109375,
+      "learning_rate": 9.58e-07,
+      "loss": 0.0006,
+      "reward": 1.7486979961395264,
+      "reward_std": 0.21525070071220398,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7643228769302368,
+      "step": 63
+    },
+    {
+      "completion_length": 119.578125,
+      "epoch": 0.08533333333333333,
+      "grad_norm": 2.5395444102381086,
+      "kl": 0.0191650390625,
+      "learning_rate": 9.573333333333333e-07,
+      "loss": 0.0008,
+      "reward": 1.843489646911621,
+      "reward_std": 0.1474801003932953,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8747395277023315,
+      "step": 64
+    },
+    {
+      "completion_length": 120.375,
+      "epoch": 0.08666666666666667,
+      "grad_norm": 2.1205975074520547,
+      "kl": 0.0228271484375,
+      "learning_rate": 9.566666666666667e-07,
+      "loss": 0.0009,
+      "reward": 1.8729538917541504,
+      "reward_std": 0.11097046732902527,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8729538917541504,
+      "step": 65
+    },
+    {
+      "completion_length": 136.328125,
+      "epoch": 0.088,
+      "grad_norm": 2.17296459266259,
+      "kl": 0.025146484375,
+      "learning_rate": 9.559999999999998e-07,
+      "loss": 0.001,
+      "reward": 1.5343749523162842,
+      "reward_std": 0.1413165032863617,
+      "rewards/format_reward": 0.890625,
+      "rewards/iou_reward": 0.643750011920929,
+      "step": 66
+    },
+    {
+      "completion_length": 120.96875,
+      "epoch": 0.08933333333333333,
+      "grad_norm": 1.4578680019756425,
+      "kl": 0.0196533203125,
+      "learning_rate": 9.553333333333334e-07,
+      "loss": 0.0008,
+      "reward": 1.8296875953674316,
+      "reward_std": 0.09358038753271103,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8296875357627869,
+      "step": 67
+    },
+    {
+      "completion_length": 105.765625,
+      "epoch": 0.09066666666666667,
+      "grad_norm": 0.7466937705267147,
+      "kl": 0.0191650390625,
+      "learning_rate": 9.546666666666665e-07,
+      "loss": 0.0008,
+      "reward": 1.8333333730697632,
+      "reward_std": 0.07788139581680298,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8489583730697632,
+      "step": 68
+    },
+    {
+      "completion_length": 117.046875,
+      "epoch": 0.092,
+      "grad_norm": 1.512434127286153,
+      "kl": 0.017333984375,
+      "learning_rate": 9.539999999999999e-07,
+      "loss": 0.0007,
+      "reward": 1.6796875,
+      "reward_std": 0.11582085490226746,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6796875,
+      "step": 69
+    },
+    {
+      "completion_length": 120.9375,
+      "epoch": 0.09333333333333334,
+      "grad_norm": 2.318530811517483,
+      "kl": 0.024658203125,
+      "learning_rate": 9.533333333333333e-07,
+      "loss": 0.001,
+      "reward": 1.6851563453674316,
+      "reward_std": 0.2016444206237793,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7007812261581421,
+      "step": 70
+    },
+    {
+      "completion_length": 120.140625,
+      "epoch": 0.09466666666666666,
+      "grad_norm": 2.3613874300299416,
+      "kl": 0.02685546875,
+      "learning_rate": 9.526666666666666e-07,
+      "loss": 0.0011,
+      "reward": 1.7236979007720947,
+      "reward_std": 0.19033756852149963,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7393229603767395,
+      "step": 71
+    },
+    {
+      "completion_length": 130.0625,
+      "epoch": 0.096,
+      "grad_norm": 1.522946021986045,
+      "kl": 0.0223388671875,
+      "learning_rate": 9.52e-07,
+      "loss": 0.0009,
+      "reward": 1.663095235824585,
+      "reward_std": 0.11986761540174484,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.709970235824585,
+      "step": 72
+    },
+    {
+      "completion_length": 124.65625,
+      "epoch": 0.09733333333333333,
+      "grad_norm": 1.9494081175469549,
+      "kl": 0.017822265625,
+      "learning_rate": 9.513333333333333e-07,
+      "loss": 0.0007,
+      "reward": 1.741964340209961,
+      "reward_std": 0.2024303376674652,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7419643402099609,
+      "step": 73
+    },
+    {
+      "completion_length": 114.546875,
+      "epoch": 0.09866666666666667,
+      "grad_norm": 2.4024979059302254,
+      "kl": 0.0262451171875,
+      "learning_rate": 9.506666666666667e-07,
+      "loss": 0.0011,
+      "reward": 1.7216145992279053,
+      "reward_std": 0.1293010413646698,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7216145992279053,
+      "step": 74
+    },
+    {
+      "completion_length": 121.984375,
+      "epoch": 0.1,
+      "grad_norm": 3.197211856596533,
+      "kl": 0.02734375,
+      "learning_rate": 9.499999999999999e-07,
+      "loss": 0.0011,
+      "reward": 1.6834635734558105,
+      "reward_std": 0.16814112663269043,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7459635734558105,
+      "step": 75
+    },
+    {
+      "completion_length": 112.96875,
+      "epoch": 0.10133333333333333,
+      "grad_norm": 3.187801047270421,
+      "kl": 0.021484375,
+      "learning_rate": 9.493333333333334e-07,
+      "loss": 0.0009,
+      "reward": 1.87890625,
+      "reward_std": 0.13995979726314545,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.89453125,
+      "step": 76
+    },
+    {
+      "completion_length": 117.921875,
+      "epoch": 0.10266666666666667,
+      "grad_norm": 4.073896522076457,
+      "kl": 0.0252685546875,
+      "learning_rate": 9.486666666666666e-07,
+      "loss": 0.001,
+      "reward": 1.6104167699813843,
+      "reward_std": 0.18849492073059082,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6260416507720947,
+      "step": 77
+    },
+    {
+      "completion_length": 125.046875,
+      "epoch": 0.104,
+      "grad_norm": 1.5689773093557249,
+      "kl": 0.021240234375,
+      "learning_rate": 9.479999999999999e-07,
+      "loss": 0.0008,
+      "reward": 1.8151042461395264,
+      "reward_std": 0.11978694051504135,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8307291269302368,
+      "step": 78
+    },
+    {
+      "completion_length": 107.390625,
+      "epoch": 0.10533333333333333,
+      "grad_norm": 4.465075361210691,
+      "kl": 0.032470703125,
+      "learning_rate": 9.473333333333333e-07,
+      "loss": 0.0013,
+      "reward": 1.6067708730697632,
+      "reward_std": 0.2259320616722107,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.6692708730697632,
+      "step": 79
+    },
+    {
+      "completion_length": 117.140625,
+      "epoch": 0.10666666666666667,
+      "grad_norm": 1.6298408135238511,
+      "kl": 0.03759765625,
+      "learning_rate": 9.466666666666666e-07,
+      "loss": 0.0015,
+      "reward": 1.8233072757720947,
+      "reward_std": 0.1297810673713684,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8233073353767395,
+      "step": 80
+    },
+    {
+      "completion_length": 121.84375,
+      "epoch": 0.108,
+      "grad_norm": 2.019192032011314,
+      "kl": 0.02099609375,
+      "learning_rate": 9.459999999999999e-07,
+      "loss": 0.0008,
+      "reward": 1.7765624523162842,
+      "reward_std": 0.12520183622837067,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7765624523162842,
+      "step": 81
+    },
+    {
+      "completion_length": 126.078125,
+      "epoch": 0.10933333333333334,
+      "grad_norm": 1.6932935564034453,
+      "kl": 0.01806640625,
+      "learning_rate": 9.453333333333333e-07,
+      "loss": 0.0007,
+      "reward": 1.7838542461395264,
+      "reward_std": 0.1805221438407898,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7994792461395264,
+      "step": 82
+    },
+    {
+      "completion_length": 115.625,
+      "epoch": 0.11066666666666666,
+      "grad_norm": 2.6921872760465475,
+      "kl": 0.0284423828125,
+      "learning_rate": 9.446666666666666e-07,
+      "loss": 0.0011,
+      "reward": 1.815941333770752,
+      "reward_std": 0.11505082994699478,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8471912145614624,
+      "step": 83
+    },
+    {
+      "completion_length": 127.25,
+      "epoch": 0.112,
+      "grad_norm": 4.0957019567290445,
+      "kl": 0.0303955078125,
+      "learning_rate": 9.439999999999999e-07,
+      "loss": 0.0012,
+      "reward": 1.6825520992279053,
+      "reward_std": 0.13762468099594116,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6981770992279053,
+      "step": 84
+    },
+    {
+      "completion_length": 104.484375,
+      "epoch": 0.11333333333333333,
+      "grad_norm": 2.2492493310243895,
+      "kl": 0.025390625,
+      "learning_rate": 9.433333333333333e-07,
+      "loss": 0.001,
+      "reward": 1.84375,
+      "reward_std": 0.11941772699356079,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8593749403953552,
+      "step": 85
+    },
+    {
+      "completion_length": 109.953125,
+      "epoch": 0.11466666666666667,
+      "grad_norm": 11.598255362229843,
+      "kl": 0.049560546875,
+      "learning_rate": 9.426666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.7864583730697632,
+      "reward_std": 0.14598765969276428,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7864583730697632,
+      "step": 86
+    },
+    {
+      "completion_length": 125.171875,
+      "epoch": 0.116,
+      "grad_norm": 1.9953078988290933,
+      "kl": 0.0302734375,
+      "learning_rate": 9.419999999999999e-07,
+      "loss": 0.0012,
+      "reward": 1.7278646230697632,
+      "reward_std": 0.1991252601146698,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7434895634651184,
+      "step": 87
+    },
+    {
+      "completion_length": 120.9375,
+      "epoch": 0.11733333333333333,
+      "grad_norm": 3.3698659329657152,
+      "kl": 0.03466796875,
+      "learning_rate": 9.413333333333333e-07,
+      "loss": 0.0014,
+      "reward": 1.8247581720352173,
+      "reward_std": 0.15209665894508362,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8403831720352173,
+      "step": 88
+    },
+    {
+      "completion_length": 117.1875,
+      "epoch": 0.11866666666666667,
+      "grad_norm": 31.655645525549655,
+      "kl": 0.040283203125,
+      "learning_rate": 9.406666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.6552083492279053,
+      "reward_std": 0.12002745270729065,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6552082896232605,
+      "step": 89
+    },
+    {
+      "completion_length": 111.703125,
+      "epoch": 0.12,
+      "grad_norm": 3.186778740786653,
+      "kl": 0.0458984375,
+      "learning_rate": 9.399999999999999e-07,
+      "loss": 0.0018,
+      "reward": 1.76171875,
+      "reward_std": 0.08353859186172485,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.76171875,
+      "step": 90
+    },
+    {
+      "completion_length": 112.078125,
+      "epoch": 0.12133333333333333,
+      "grad_norm": 2.389976800936232,
+      "kl": 0.033447265625,
+      "learning_rate": 9.393333333333334e-07,
+      "loss": 0.0013,
+      "reward": 1.770721673965454,
+      "reward_std": 0.11786292493343353,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7707216739654541,
+      "step": 91
+    },
+    {
+      "completion_length": 110.015625,
+      "epoch": 0.12266666666666666,
+      "grad_norm": 7.9604374705002945,
+      "kl": 0.0279541015625,
+      "learning_rate": 9.386666666666666e-07,
+      "loss": 0.0011,
+      "reward": 1.6351561546325684,
+      "reward_std": 0.12967410683631897,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6351562738418579,
+      "step": 92
+    },
+    {
+      "completion_length": 111.390625,
+      "epoch": 0.124,
+      "grad_norm": 2.211172019232313,
+      "kl": 0.03662109375,
+      "learning_rate": 9.379999999999998e-07,
+      "loss": 0.0015,
+      "reward": 1.7682292461395264,
+      "reward_std": 0.19485574960708618,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.8307291269302368,
+      "step": 93
+    },
+    {
+      "completion_length": 128.078125,
+      "epoch": 0.12533333333333332,
+      "grad_norm": 2.454821827380843,
+      "kl": 0.041748046875,
+      "learning_rate": 9.373333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.7106027603149414,
+      "reward_std": 0.14825835824012756,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7106027007102966,
+      "step": 94
+    },
+    {
+      "completion_length": 135.921875,
+      "epoch": 0.12666666666666668,
+      "grad_norm": 3.09424535867531,
+      "kl": 0.0262451171875,
+      "learning_rate": 9.366666666666666e-07,
+      "loss": 0.0011,
+      "reward": 1.7659733295440674,
+      "reward_std": 0.12523730099201202,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7815983891487122,
+      "step": 95
+    },
+    {
+      "completion_length": 109.671875,
+      "epoch": 0.128,
+      "grad_norm": 4.6011374805182985,
+      "kl": 0.044189453125,
+      "learning_rate": 9.36e-07,
+      "loss": 0.0018,
+      "reward": 1.8533110618591309,
+      "reward_std": 0.0976390540599823,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8533110618591309,
+      "step": 96
+    },
+    {
+      "completion_length": 122.28125,
+      "epoch": 0.12933333333333333,
+      "grad_norm": 3.563018914139395,
+      "kl": 0.049072265625,
+      "learning_rate": 9.353333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.6610491275787354,
+      "reward_std": 0.14633873105049133,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6610491275787354,
+      "step": 97
+    },
+    {
+      "completion_length": 118.359375,
+      "epoch": 0.13066666666666665,
+      "grad_norm": 1.8369144583440116,
+      "kl": 0.04248046875,
+      "learning_rate": 9.346666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.808333396911621,
+      "reward_std": 0.06176029145717621,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8239583969116211,
+      "step": 98
+    },
+    {
+      "completion_length": 111.578125,
+      "epoch": 0.132,
+      "grad_norm": 1.8727990918460289,
+      "kl": 0.038818359375,
+      "learning_rate": 9.34e-07,
+      "loss": 0.0016,
+      "reward": 1.8588913679122925,
+      "reward_std": 0.04047934710979462,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8588913679122925,
+      "step": 99
+    },
+    {
+      "completion_length": 107.359375,
+      "epoch": 0.13333333333333333,
+      "grad_norm": 84.88553935088964,
+      "kl": 0.04541015625,
+      "learning_rate": 9.333333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.7843750715255737,
+      "reward_std": 0.09295584261417389,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.784375011920929,
+      "step": 100
+    },
+    {
+      "completion_length": 132.171875,
+      "epoch": 0.13466666666666666,
+      "grad_norm": 3.856845762421409,
+      "kl": 0.033935546875,
+      "learning_rate": 9.326666666666666e-07,
+      "loss": 0.0014,
+      "reward": 1.8221354484558105,
+      "reward_std": 0.15216070413589478,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8221353888511658,
+      "step": 101
+    },
+    {
+      "completion_length": 118.0625,
+      "epoch": 0.136,
+      "grad_norm": 1.5970701522273376,
+      "kl": 0.049072265625,
+      "learning_rate": 9.32e-07,
+      "loss": 0.002,
+      "reward": 1.8090401887893677,
+      "reward_std": 0.08907502144575119,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8246651887893677,
+      "step": 102
+    },
+    {
+      "completion_length": 122.78125,
+      "epoch": 0.13733333333333334,
+      "grad_norm": 1.8943114480297685,
+      "kl": 0.0174560546875,
+      "learning_rate": 9.313333333333333e-07,
+      "loss": 0.0007,
+      "reward": 1.9139137268066406,
+      "reward_std": 0.11439535766839981,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9295386672019958,
+      "step": 103
+    },
+    {
+      "completion_length": 110.21875,
+      "epoch": 0.13866666666666666,
+      "grad_norm": 2.7360915800538144,
+      "kl": 0.0390625,
+      "learning_rate": 9.306666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.7588541507720947,
+      "reward_std": 0.10715585947036743,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7588542103767395,
+      "step": 104
+    },
+    {
+      "completion_length": 109.078125,
+      "epoch": 0.14,
+      "grad_norm": 2.7186391515482553,
+      "kl": 0.06494140625,
+      "learning_rate": 9.3e-07,
+      "loss": 0.0026,
+      "reward": 1.7854167222976685,
+      "reward_std": 0.05191417410969734,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7854167222976685,
+      "step": 105
+    },
+    {
+      "completion_length": 132.453125,
+      "epoch": 0.14133333333333334,
+      "grad_norm": 2.8637572638241435,
+      "kl": 0.041748046875,
+      "learning_rate": 9.293333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.5825520753860474,
+      "reward_std": 0.27541089057922363,
+      "rewards/format_reward": 0.890625,
+      "rewards/iou_reward": 0.6919270753860474,
+      "step": 106
+    },
+    {
+      "completion_length": 135.375,
+      "epoch": 0.14266666666666666,
+      "grad_norm": 2.8465703718056545,
+      "kl": 0.04736328125,
+      "learning_rate": 9.286666666666666e-07,
+      "loss": 0.0019,
+      "reward": 1.7020833492279053,
+      "reward_std": 0.21091903746128082,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7489582896232605,
+      "step": 107
+    },
+    {
+      "completion_length": 108.109375,
+      "epoch": 0.144,
+      "grad_norm": 2.1057350108207,
+      "kl": 0.037353515625,
+      "learning_rate": 9.28e-07,
+      "loss": 0.0015,
+      "reward": 1.7630208730697632,
+      "reward_std": 0.1398771107196808,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7630208134651184,
+      "step": 108
+    },
+    {
+      "completion_length": 121.90625,
+      "epoch": 0.14533333333333334,
+      "grad_norm": 2.861950080549418,
+      "kl": 0.05126953125,
+      "learning_rate": 9.273333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.7861979007720947,
+      "reward_std": 0.18718063831329346,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.8643229007720947,
+      "step": 109
+    },
+    {
+      "completion_length": 118.46875,
+      "epoch": 0.14666666666666667,
+      "grad_norm": 8.683605670145088,
+      "kl": 0.044677734375,
+      "learning_rate": 9.266666666666665e-07,
+      "loss": 0.0018,
+      "reward": 1.7252604961395264,
+      "reward_std": 0.10607947409152985,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7252603769302368,
+      "step": 110
+    },
+    {
+      "completion_length": 107.0625,
+      "epoch": 0.148,
+      "grad_norm": 1.8014247213922776,
+      "kl": 0.041259765625,
+      "learning_rate": 9.26e-07,
+      "loss": 0.0016,
+      "reward": 1.8854167461395264,
+      "reward_std": 0.0833333283662796,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8854166865348816,
+      "step": 111
+    },
+    {
+      "completion_length": 140.578125,
+      "epoch": 0.14933333333333335,
+      "grad_norm": 1.895349572030504,
+      "kl": 0.040771484375,
+      "learning_rate": 9.253333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.6875,
+      "reward_std": 0.31290408968925476,
+      "rewards/format_reward": 0.90625,
+      "rewards/iou_reward": 0.78125,
+      "step": 112
+    },
+    {
+      "completion_length": 108.953125,
+      "epoch": 0.15066666666666667,
+      "grad_norm": 2.6276706129173926,
+      "kl": 0.03759765625,
+      "learning_rate": 9.246666666666666e-07,
+      "loss": 0.0015,
+      "reward": 1.8283854722976685,
+      "reward_std": 0.0892583504319191,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8283854126930237,
+      "step": 113
+    },
+    {
+      "completion_length": 108.078125,
+      "epoch": 0.152,
+      "grad_norm": 2.2465681877670804,
+      "kl": 0.05810546875,
+      "learning_rate": 9.24e-07,
+      "loss": 0.0023,
+      "reward": 1.7622767686843872,
+      "reward_std": 0.04984360933303833,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7622767686843872,
+      "step": 114
+    },
+    {
+      "completion_length": 118.46875,
+      "epoch": 0.15333333333333332,
+      "grad_norm": 1.7153109268430322,
+      "kl": 0.030029296875,
+      "learning_rate": 9.233333333333333e-07,
+      "loss": 0.0012,
+      "reward": 1.689843773841858,
+      "reward_std": 0.19731566309928894,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7210937738418579,
+      "step": 115
+    },
+    {
+      "completion_length": 119.5625,
+      "epoch": 0.15466666666666667,
+      "grad_norm": 2.344346356417858,
+      "kl": 0.03369140625,
+      "learning_rate": 9.226666666666666e-07,
+      "loss": 0.0013,
+      "reward": 1.8723958730697632,
+      "reward_std": 0.09119774401187897,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8880208730697632,
+      "step": 116
+    },
+    {
+      "completion_length": 118.6875,
+      "epoch": 0.156,
+      "grad_norm": 2.6247247521599704,
+      "kl": 0.048583984375,
+      "learning_rate": 9.22e-07,
+      "loss": 0.0019,
+      "reward": 1.78125,
+      "reward_std": 0.15751445293426514,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8125000596046448,
+      "step": 117
+    },
+    {
+      "completion_length": 116.25,
+      "epoch": 0.15733333333333333,
+      "grad_norm": 2.001726424710619,
+      "kl": 0.0390625,
+      "learning_rate": 9.213333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.8716145753860474,
+      "reward_std": 0.16429051756858826,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.9028645753860474,
+      "step": 118
+    },
+    {
+      "completion_length": 135.28125,
+      "epoch": 0.15866666666666668,
+      "grad_norm": 4.543068266979912,
+      "kl": 0.033447265625,
+      "learning_rate": 9.206666666666666e-07,
+      "loss": 0.0013,
+      "reward": 1.7674479484558105,
+      "reward_std": 0.16686108708381653,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8143229484558105,
+      "step": 119
+    },
+    {
+      "completion_length": 124.78125,
+      "epoch": 0.16,
+      "grad_norm": 2.7177354991708973,
+      "kl": 0.042724609375,
+      "learning_rate": 9.2e-07,
+      "loss": 0.0017,
+      "reward": 1.7330729961395264,
+      "reward_std": 0.11086425930261612,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7486979365348816,
+      "step": 120
+    },
+    {
+      "completion_length": 124.65625,
+      "epoch": 0.16133333333333333,
+      "grad_norm": 2.754443395405747,
+      "kl": 0.02685546875,
+      "learning_rate": 9.193333333333333e-07,
+      "loss": 0.0011,
+      "reward": 1.6875,
+      "reward_std": 0.22300985455513,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.75,
+      "step": 121
+    },
+    {
+      "completion_length": 109.375,
+      "epoch": 0.16266666666666665,
+      "grad_norm": 0.9568118813331301,
+      "kl": 0.044921875,
+      "learning_rate": 9.186666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.8526041507720947,
+      "reward_std": 0.0031249995809048414,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8526042103767395,
+      "step": 122
+    },
+    {
+      "completion_length": 114.203125,
+      "epoch": 0.164,
+      "grad_norm": 2.9504765632900156,
+      "kl": 0.0625,
+      "learning_rate": 9.18e-07,
+      "loss": 0.0025,
+      "reward": 1.8162202835083008,
+      "reward_std": 0.16511648893356323,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8162202835083008,
+      "step": 123
+    },
+    {
+      "completion_length": 117.75,
+      "epoch": 0.16533333333333333,
+      "grad_norm": 3.250454911296537,
+      "kl": 0.0458984375,
+      "learning_rate": 9.173333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.7484374046325684,
+      "reward_std": 0.13522613048553467,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7484374642372131,
+      "step": 124
+    },
+    {
+      "completion_length": 120.03125,
+      "epoch": 0.16666666666666666,
+      "grad_norm": 1.6387570499443573,
+      "kl": 0.032470703125,
+      "learning_rate": 9.166666666666665e-07,
+      "loss": 0.0013,
+      "reward": 1.753645896911621,
+      "reward_std": 0.14578112959861755,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7692708373069763,
+      "step": 125
+    },
+    {
+      "completion_length": 118.1875,
+      "epoch": 0.168,
+      "grad_norm": 2.161309979322781,
+      "kl": 0.06591796875,
+      "learning_rate": 9.16e-07,
+      "loss": 0.0026,
+      "reward": 1.7233631610870361,
+      "reward_std": 0.10922516137361526,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7233631014823914,
+      "step": 126
+    },
+    {
+      "completion_length": 126.796875,
+      "epoch": 0.16933333333333334,
+      "grad_norm": 4.710207570843693,
+      "kl": 0.046142578125,
+      "learning_rate": 9.153333333333332e-07,
+      "loss": 0.0018,
+      "reward": 1.662500023841858,
+      "reward_std": 0.21690717339515686,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6781250238418579,
+      "step": 127
+    },
+    {
+      "completion_length": 119.96875,
+      "epoch": 0.17066666666666666,
+      "grad_norm": 1.6044244492338207,
+      "kl": 0.056884765625,
+      "learning_rate": 9.146666666666666e-07,
+      "loss": 0.0023,
+      "reward": 1.7614026069641113,
+      "reward_std": 0.08900903165340424,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7614025473594666,
+      "step": 128
+    },
+    {
+      "completion_length": 117.59375,
+      "epoch": 0.172,
+      "grad_norm": 0.590615243186772,
+      "kl": 0.02001953125,
+      "learning_rate": 9.14e-07,
+      "loss": 0.0008,
+      "reward": 1.9010417461395264,
+      "reward_std": 0.03125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9010416865348816,
+      "step": 129
+    },
+    {
+      "completion_length": 117.15625,
+      "epoch": 0.17333333333333334,
+      "grad_norm": 1.9455015561856543,
+      "kl": 0.04296875,
+      "learning_rate": 9.133333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.8687763214111328,
+      "reward_std": 0.16835281252861023,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8844013214111328,
+      "step": 130
+    },
+    {
+      "completion_length": 121.28125,
+      "epoch": 0.17466666666666666,
+      "grad_norm": 1.527211020946494,
+      "kl": 0.040771484375,
+      "learning_rate": 9.126666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.8096354007720947,
+      "reward_std": 0.07616984099149704,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8252604603767395,
+      "step": 131
+    },
+    {
+      "completion_length": 122.9375,
+      "epoch": 0.176,
+      "grad_norm": 5.360996775160784,
+      "kl": 0.039794921875,
+      "learning_rate": 9.12e-07,
+      "loss": 0.0016,
+      "reward": 1.8166667222976685,
+      "reward_std": 0.21562500298023224,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8479166626930237,
+      "step": 132
+    },
+    {
+      "completion_length": 117.25,
+      "epoch": 0.17733333333333334,
+      "grad_norm": 1.9501931690204812,
+      "kl": 0.051025390625,
+      "learning_rate": 9.113333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.7748898267745972,
+      "reward_std": 0.11519991606473923,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7905148267745972,
+      "step": 133
+    },
+    {
+      "completion_length": 113.125,
+      "epoch": 0.17866666666666667,
+      "grad_norm": 2.5602196917640994,
+      "kl": 0.060546875,
+      "learning_rate": 9.106666666666666e-07,
+      "loss": 0.0024,
+      "reward": 1.7504465579986572,
+      "reward_std": 0.059710822999477386,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7504464983940125,
+      "step": 134
+    },
+    {
+      "completion_length": 126.890625,
+      "epoch": 0.18,
+      "grad_norm": 2.238025497642626,
+      "kl": 0.038330078125,
+      "learning_rate": 9.1e-07,
+      "loss": 0.0015,
+      "reward": 1.632552146911621,
+      "reward_std": 0.23623128235340118,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6481771469116211,
+      "step": 135
+    },
+    {
+      "completion_length": 128.59375,
+      "epoch": 0.18133333333333335,
+      "grad_norm": 3.29183069447896,
+      "kl": 0.049072265625,
+      "learning_rate": 9.093333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.7192150354385376,
+      "reward_std": 0.13154228031635284,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7504649758338928,
+      "step": 136
+    },
+    {
+      "completion_length": 123.9375,
+      "epoch": 0.18266666666666667,
+      "grad_norm": 1.4140864052256175,
+      "kl": 0.04052734375,
+      "learning_rate": 9.086666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.8500744104385376,
+      "reward_std": 0.09614831209182739,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8500744104385376,
+      "step": 137
+    },
+    {
+      "completion_length": 108.1875,
+      "epoch": 0.184,
+      "grad_norm": 4.039189528790454,
+      "kl": 0.03369140625,
+      "learning_rate": 9.08e-07,
+      "loss": 0.0013,
+      "reward": 1.841406226158142,
+      "reward_std": 0.13452188670635223,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8414062261581421,
+      "step": 138
+    },
+    {
+      "completion_length": 119.53125,
+      "epoch": 0.18533333333333332,
+      "grad_norm": 2.6677204639450025,
+      "kl": 0.056884765625,
+      "learning_rate": 9.073333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.8539061546325684,
+      "reward_std": 0.06832996755838394,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8539062738418579,
+      "step": 139
+    },
+    {
+      "completion_length": 118.40625,
+      "epoch": 0.18666666666666668,
+      "grad_norm": 1.8966737958529576,
+      "kl": 0.05810546875,
+      "learning_rate": 9.066666666666665e-07,
+      "loss": 0.0023,
+      "reward": 1.8138021230697632,
+      "reward_std": 0.10585569590330124,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8294271230697632,
+      "step": 140
+    },
+    {
+      "completion_length": 121.046875,
+      "epoch": 0.188,
+      "grad_norm": 1.4325431183048911,
+      "kl": 0.036376953125,
+      "learning_rate": 9.06e-07,
+      "loss": 0.0015,
+      "reward": 1.6510417461395264,
+      "reward_std": 0.17984378337860107,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7135416865348816,
+      "step": 141
+    },
+    {
+      "completion_length": 124.484375,
+      "epoch": 0.18933333333333333,
+      "grad_norm": 5.955021285439134,
+      "kl": 0.042724609375,
+      "learning_rate": 9.053333333333332e-07,
+      "loss": 0.0017,
+      "reward": 1.7335751056671143,
+      "reward_std": 0.2471698522567749,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7804501056671143,
+      "step": 142
+    },
+    {
+      "completion_length": 113.953125,
+      "epoch": 0.19066666666666668,
+      "grad_norm": 3.385493663839884,
+      "kl": 0.041015625,
+      "learning_rate": 9.046666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.65625,
+      "reward_std": 0.18044094741344452,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.65625,
+      "step": 143
+    },
+    {
+      "completion_length": 129.609375,
+      "epoch": 0.192,
+      "grad_norm": 2.7457947419424684,
+      "kl": 0.0498046875,
+      "learning_rate": 9.039999999999999e-07,
+      "loss": 0.002,
+      "reward": 1.736718773841858,
+      "reward_std": 0.14740490913391113,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7367187738418579,
+      "step": 144
+    },
+    {
+      "completion_length": 111.15625,
+      "epoch": 0.19333333333333333,
+      "grad_norm": 3.103797915879443,
+      "kl": 0.06201171875,
+      "learning_rate": 9.033333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.655989646911621,
+      "reward_std": 0.14173895120620728,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6559895873069763,
+      "step": 145
+    },
+    {
+      "completion_length": 119.46875,
+      "epoch": 0.19466666666666665,
+      "grad_norm": 4.964234993443217,
+      "kl": 0.054443359375,
+      "learning_rate": 9.026666666666665e-07,
+      "loss": 0.0022,
+      "reward": 1.6739583015441895,
+      "reward_std": 0.11833992600440979,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6895833611488342,
+      "step": 146
+    },
+    {
+      "completion_length": 122.171875,
+      "epoch": 0.196,
+      "grad_norm": 2.4878588159179826,
+      "kl": 0.04443359375,
+      "learning_rate": 9.02e-07,
+      "loss": 0.0018,
+      "reward": 1.6985559463500977,
+      "reward_std": 0.14896918833255768,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6985558867454529,
+      "step": 147
+    },
+    {
+      "completion_length": 118.78125,
+      "epoch": 0.19733333333333333,
+      "grad_norm": 11.293845021513935,
+      "kl": 0.058349609375,
+      "learning_rate": 9.013333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.7135417461395264,
+      "reward_std": 0.14046694338321686,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7135416865348816,
+      "step": 148
+    },
+    {
+      "completion_length": 116.484375,
+      "epoch": 0.19866666666666666,
+      "grad_norm": 1.6066614517692293,
+      "kl": 0.0419921875,
+      "learning_rate": 9.006666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.5705729722976685,
+      "reward_std": 0.13704469799995422,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.5861979722976685,
+      "step": 149
+    },
+    {
+      "completion_length": 135.59375,
+      "epoch": 0.2,
+      "grad_norm": 2.1383650627856383,
+      "kl": 0.03173828125,
+      "learning_rate": 9e-07,
+      "loss": 0.0013,
+      "reward": 1.7234747409820557,
+      "reward_std": 0.25432926416397095,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7547246813774109,
+      "step": 150
+    },
+    {
+      "completion_length": 119.90625,
+      "epoch": 0.20133333333333334,
+      "grad_norm": 1.8969662594386645,
+      "kl": 0.04443359375,
+      "learning_rate": 8.993333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.7356771230697632,
+      "reward_std": 0.0836520567536354,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7513020634651184,
+      "step": 151
+    },
+    {
+      "completion_length": 125.96875,
+      "epoch": 0.20266666666666666,
+      "grad_norm": 1.4344238372064497,
+      "kl": 0.0223388671875,
+      "learning_rate": 8.986666666666666e-07,
+      "loss": 0.0009,
+      "reward": 1.8114583492279053,
+      "reward_std": 0.13180916011333466,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8270833492279053,
+      "step": 152
+    },
+    {
+      "completion_length": 112.34375,
+      "epoch": 0.204,
+      "grad_norm": 3.229051393388542,
+      "kl": 0.060791015625,
+      "learning_rate": 8.98e-07,
+      "loss": 0.0024,
+      "reward": 1.749739646911621,
+      "reward_std": 0.1689063012599945,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7653645277023315,
+      "step": 153
+    },
+    {
+      "completion_length": 116.75,
+      "epoch": 0.20533333333333334,
+      "grad_norm": 2.9628055108483533,
+      "kl": 0.059326171875,
+      "learning_rate": 8.973333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.729836344718933,
+      "reward_std": 0.12505482137203217,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7454613447189331,
+      "step": 154
+    },
+    {
+      "completion_length": 132.828125,
+      "epoch": 0.20666666666666667,
+      "grad_norm": 2.064245439737039,
+      "kl": 0.04345703125,
+      "learning_rate": 8.966666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.7124345302581787,
+      "reward_std": 0.2601771950721741,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7436845898628235,
+      "step": 155
+    },
+    {
+      "completion_length": 126.75,
+      "epoch": 0.208,
+      "grad_norm": 2.0011413117964194,
+      "kl": 0.055908203125,
+      "learning_rate": 8.96e-07,
+      "loss": 0.0022,
+      "reward": 1.677343726158142,
+      "reward_std": 0.11880233883857727,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6929687857627869,
+      "step": 156
+    },
+    {
+      "completion_length": 111.5,
+      "epoch": 0.20933333333333334,
+      "grad_norm": 2.397413187259402,
+      "kl": 0.07373046875,
+      "learning_rate": 8.953333333333332e-07,
+      "loss": 0.003,
+      "reward": 1.8601562976837158,
+      "reward_std": 0.06388392299413681,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8601562976837158,
+      "step": 157
+    },
+    {
+      "completion_length": 115.3125,
+      "epoch": 0.21066666666666667,
+      "grad_norm": 1.607148995254747,
+      "kl": 0.037109375,
+      "learning_rate": 8.946666666666667e-07,
+      "loss": 0.0015,
+      "reward": 1.9114583730697632,
+      "reward_std": 0.12102918326854706,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9270833730697632,
+      "step": 158
+    },
+    {
+      "completion_length": 123.46875,
+      "epoch": 0.212,
+      "grad_norm": 14.983304441321664,
+      "kl": 0.068359375,
+      "learning_rate": 8.939999999999999e-07,
+      "loss": 0.0027,
+      "reward": 1.7553199529647827,
+      "reward_std": 0.15713472664356232,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7709449529647827,
+      "step": 159
+    },
+    {
+      "completion_length": 129.390625,
+      "epoch": 0.21333333333333335,
+      "grad_norm": 2.2820914392535827,
+      "kl": 0.05712890625,
+      "learning_rate": 8.933333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.655989646911621,
+      "reward_std": 0.18078705668449402,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7028646469116211,
+      "step": 160
+    },
+    {
+      "completion_length": 128.0625,
+      "epoch": 0.21466666666666667,
+      "grad_norm": 1.5608750146561887,
+      "kl": 0.0693359375,
+      "learning_rate": 8.926666666666666e-07,
+      "loss": 0.0028,
+      "reward": 1.684114694595337,
+      "reward_std": 0.09522302448749542,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.7622395753860474,
+      "step": 161
+    },
+    {
+      "completion_length": 115.421875,
+      "epoch": 0.216,
+      "grad_norm": 1.92307712087476,
+      "kl": 0.0308837890625,
+      "learning_rate": 8.92e-07,
+      "loss": 0.0012,
+      "reward": 1.9479167461395264,
+      "reward_std": 0.1041666641831398,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9635416865348816,
+      "step": 162
+    },
+    {
+      "completion_length": 134.671875,
+      "epoch": 0.21733333333333332,
+      "grad_norm": 2.1307660125874524,
+      "kl": 0.040283203125,
+      "learning_rate": 8.913333333333332e-07,
+      "loss": 0.0016,
+      "reward": 1.810937523841858,
+      "reward_std": 0.05832270532846451,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8265624642372131,
+      "step": 163
+    },
+    {
+      "completion_length": 128.65625,
+      "epoch": 0.21866666666666668,
+      "grad_norm": 3.0359144679973182,
+      "kl": 0.040771484375,
+      "learning_rate": 8.906666666666667e-07,
+      "loss": 0.0016,
+      "reward": 1.705468773841858,
+      "reward_std": 0.2289583832025528,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7523437738418579,
+      "step": 164
+    },
+    {
+      "completion_length": 122.546875,
+      "epoch": 0.22,
+      "grad_norm": 2.165026718844832,
+      "kl": 0.055419921875,
+      "learning_rate": 8.9e-07,
+      "loss": 0.0022,
+      "reward": 1.7838542461395264,
+      "reward_std": 0.0989583283662796,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7994792461395264,
+      "step": 165
+    },
+    {
+      "completion_length": 115.96875,
+      "epoch": 0.22133333333333333,
+      "grad_norm": 1.306417545935995,
+      "kl": 0.04443359375,
+      "learning_rate": 8.893333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.840364694595337,
+      "reward_std": 0.0291642677038908,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8403646349906921,
+      "step": 166
+    },
+    {
+      "completion_length": 124.546875,
+      "epoch": 0.22266666666666668,
+      "grad_norm": 1.803388082724576,
+      "kl": 0.0322265625,
+      "learning_rate": 8.886666666666667e-07,
+      "loss": 0.0013,
+      "reward": 1.8661458492279053,
+      "reward_std": 0.14623497426509857,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8661458492279053,
+      "step": 167
+    },
+    {
+      "completion_length": 129.0,
+      "epoch": 0.224,
+      "grad_norm": 1.8047819238697869,
+      "kl": 0.037353515625,
+      "learning_rate": 8.88e-07,
+      "loss": 0.0015,
+      "reward": 1.7033109664916992,
+      "reward_std": 0.1748872995376587,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7345609664916992,
+      "step": 168
+    },
+    {
+      "completion_length": 133.109375,
+      "epoch": 0.22533333333333333,
+      "grad_norm": 1.2654613100316443,
+      "kl": 0.043701171875,
+      "learning_rate": 8.873333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.765625,
+      "reward_std": 0.14317253232002258,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.8437500596046448,
+      "step": 169
+    },
+    {
+      "completion_length": 124.34375,
+      "epoch": 0.22666666666666666,
+      "grad_norm": 3.0101784436467702,
+      "kl": 0.0712890625,
+      "learning_rate": 8.866666666666667e-07,
+      "loss": 0.0028,
+      "reward": 1.7814359664916992,
+      "reward_std": 0.13738565146923065,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.781436026096344,
+      "step": 170
+    },
+    {
+      "completion_length": 116.671875,
+      "epoch": 0.228,
+      "grad_norm": 9.93128173314353,
+      "kl": 0.049560546875,
+      "learning_rate": 8.86e-07,
+      "loss": 0.002,
+      "reward": 1.881250023841858,
+      "reward_std": 0.13774332404136658,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8812499642372131,
+      "step": 171
+    },
+    {
+      "completion_length": 120.34375,
+      "epoch": 0.22933333333333333,
+      "grad_norm": 2.060124974214225,
+      "kl": 0.055419921875,
+      "learning_rate": 8.853333333333332e-07,
+      "loss": 0.0022,
+      "reward": 1.658783197402954,
+      "reward_std": 0.17403516173362732,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7056581377983093,
+      "step": 172
+    },
+    {
+      "completion_length": 131.609375,
+      "epoch": 0.23066666666666666,
+      "grad_norm": 1.0162899533477452,
+      "kl": 0.025146484375,
+      "learning_rate": 8.846666666666667e-07,
+      "loss": 0.001,
+      "reward": 1.8203125,
+      "reward_std": 0.10494468361139297,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8671875,
+      "step": 173
+    },
+    {
+      "completion_length": 116.5625,
+      "epoch": 0.232,
+      "grad_norm": 1.6310960683895646,
+      "kl": 0.05517578125,
+      "learning_rate": 8.839999999999999e-07,
+      "loss": 0.0022,
+      "reward": 1.8567708730697632,
+      "reward_std": 0.12623751163482666,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8567708134651184,
+      "step": 174
+    },
+    {
+      "completion_length": 126.140625,
+      "epoch": 0.23333333333333334,
+      "grad_norm": 2.001532260500075,
+      "kl": 0.045654296875,
+      "learning_rate": 8.833333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.8375000953674316,
+      "reward_std": 0.07452811300754547,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8374999761581421,
+      "step": 175
+    },
+    {
+      "completion_length": 118.671875,
+      "epoch": 0.23466666666666666,
+      "grad_norm": 2.012364738195064,
+      "kl": 0.05029296875,
+      "learning_rate": 8.826666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.762239694595337,
+      "reward_std": 0.1282718926668167,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7622395753860474,
+      "step": 176
+    },
+    {
+      "completion_length": 137.671875,
+      "epoch": 0.236,
+      "grad_norm": 2.4454923513592886,
+      "kl": 0.06591796875,
+      "learning_rate": 8.82e-07,
+      "loss": 0.0026,
+      "reward": 1.5923477411270142,
+      "reward_std": 0.22222742438316345,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.6392227411270142,
+      "step": 177
+    },
+    {
+      "completion_length": 136.3125,
+      "epoch": 0.23733333333333334,
+      "grad_norm": 1.6005532775616316,
+      "kl": 0.044189453125,
+      "learning_rate": 8.813333333333332e-07,
+      "loss": 0.0018,
+      "reward": 1.6809896230697632,
+      "reward_std": 0.21692714095115662,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7434895634651184,
+      "step": 178
+    },
+    {
+      "completion_length": 116.171875,
+      "epoch": 0.23866666666666667,
+      "grad_norm": 4.442276926906464,
+      "kl": 0.0712890625,
+      "learning_rate": 8.806666666666667e-07,
+      "loss": 0.0028,
+      "reward": 1.616927146911621,
+      "reward_std": 0.13878649473190308,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6169270873069763,
+      "step": 179
+    },
+    {
+      "completion_length": 118.328125,
+      "epoch": 0.24,
+      "grad_norm": 2.6197505186750716,
+      "kl": 0.056640625,
+      "learning_rate": 8.799999999999999e-07,
+      "loss": 0.0023,
+      "reward": 1.691145896911621,
+      "reward_std": 0.17552971839904785,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7067708969116211,
+      "step": 180
+    },
+    {
+      "completion_length": 113.03125,
+      "epoch": 0.24133333333333334,
+      "grad_norm": 1.9210425783395924,
+      "kl": 0.06787109375,
+      "learning_rate": 8.793333333333333e-07,
+      "loss": 0.0027,
+      "reward": 1.8905506134033203,
+      "reward_std": 0.02479490265250206,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8905506134033203,
+      "step": 181
+    },
+    {
+      "completion_length": 105.078125,
+      "epoch": 0.24266666666666667,
+      "grad_norm": 3.6491209221733225,
+      "kl": 0.0264892578125,
+      "learning_rate": 8.786666666666666e-07,
+      "loss": 0.0011,
+      "reward": 1.9114583730697632,
+      "reward_std": 0.08258544653654099,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9114582538604736,
+      "step": 182
+    },
+    {
+      "completion_length": 126.671875,
+      "epoch": 0.244,
+      "grad_norm": 4.487358471654779,
+      "kl": 0.0380859375,
+      "learning_rate": 8.78e-07,
+      "loss": 0.0015,
+      "reward": 1.779761791229248,
+      "reward_std": 0.0803571417927742,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7797619104385376,
+      "step": 183
+    },
+    {
+      "completion_length": 118.453125,
+      "epoch": 0.24533333333333332,
+      "grad_norm": 1.801353615915058,
+      "kl": 0.037353515625,
+      "learning_rate": 8.773333333333332e-07,
+      "loss": 0.0015,
+      "reward": 1.9070312976837158,
+      "reward_std": 0.0993998646736145,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9070312976837158,
+      "step": 184
+    },
+    {
+      "completion_length": 116.765625,
+      "epoch": 0.24666666666666667,
+      "grad_norm": 1.79135885713144,
+      "kl": 0.07373046875,
+      "learning_rate": 8.766666666666667e-07,
+      "loss": 0.003,
+      "reward": 1.7276041507720947,
+      "reward_std": 0.13925239443778992,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7276042103767395,
+      "step": 185
+    },
+    {
+      "completion_length": 122.703125,
+      "epoch": 0.248,
+      "grad_norm": 2.224805407957972,
+      "kl": 0.034912109375,
+      "learning_rate": 8.76e-07,
+      "loss": 0.0014,
+      "reward": 1.7080729007720947,
+      "reward_std": 0.10390310734510422,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7236979007720947,
+      "step": 186
+    },
+    {
+      "completion_length": 128.46875,
+      "epoch": 0.24933333333333332,
+      "grad_norm": 1.2494439073956594,
+      "kl": 0.029296875,
+      "learning_rate": 8.753333333333332e-07,
+      "loss": 0.0012,
+      "reward": 1.84375,
+      "reward_std": 0.14038139581680298,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.875,
+      "step": 187
+    },
+    {
+      "completion_length": 120.640625,
+      "epoch": 0.25066666666666665,
+      "grad_norm": 2.437236046909177,
+      "kl": 0.047607421875,
+      "learning_rate": 8.746666666666667e-07,
+      "loss": 0.0019,
+      "reward": 1.7489583492279053,
+      "reward_std": 0.17658765614032745,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7802082896232605,
+      "step": 188
+    },
+    {
+      "completion_length": 120.046875,
+      "epoch": 0.252,
+      "grad_norm": 1.6238623701329356,
+      "kl": 0.0439453125,
+      "learning_rate": 8.739999999999999e-07,
+      "loss": 0.0018,
+      "reward": 1.8111979961395264,
+      "reward_std": 0.0859375,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8111979365348816,
+      "step": 189
+    },
+    {
+      "completion_length": 120.78125,
+      "epoch": 0.25333333333333335,
+      "grad_norm": 1.5781657616688045,
+      "kl": 0.04736328125,
+      "learning_rate": 8.733333333333333e-07,
+      "loss": 0.0019,
+      "reward": 1.8058035373687744,
+      "reward_std": 0.13310521841049194,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8214285373687744,
+      "step": 190
+    },
+    {
+      "completion_length": 132.71875,
+      "epoch": 0.25466666666666665,
+      "grad_norm": 1.6449063357714848,
+      "kl": 0.043701171875,
+      "learning_rate": 8.726666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.770052194595337,
+      "reward_std": 0.08210469782352448,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7700520753860474,
+      "step": 191
+    },
+    {
+      "completion_length": 127.375,
+      "epoch": 0.256,
+      "grad_norm": 9.612171284932465,
+      "kl": 0.049560546875,
+      "learning_rate": 8.72e-07,
+      "loss": 0.002,
+      "reward": 1.8213541507720947,
+      "reward_std": 0.1122979000210762,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8213541507720947,
+      "step": 192
+    },
+    {
+      "completion_length": 117.28125,
+      "epoch": 0.25733333333333336,
+      "grad_norm": 1.6207213302561483,
+      "kl": 0.036376953125,
+      "learning_rate": 8.713333333333332e-07,
+      "loss": 0.0015,
+      "reward": 1.8125,
+      "reward_std": 0.078125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8125,
+      "step": 193
+    },
+    {
+      "completion_length": 122.234375,
+      "epoch": 0.25866666666666666,
+      "grad_norm": 1.6925953343061009,
+      "kl": 0.052490234375,
+      "learning_rate": 8.706666666666667e-07,
+      "loss": 0.0021,
+      "reward": 1.7927827835083008,
+      "reward_std": 0.13119414448738098,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7927827835083008,
+      "step": 194
+    },
+    {
+      "completion_length": 127.484375,
+      "epoch": 0.26,
+      "grad_norm": 2.4398213896803504,
+      "kl": 0.03515625,
+      "learning_rate": 8.699999999999999e-07,
+      "loss": 0.0014,
+      "reward": 1.8424479961395264,
+      "reward_std": 0.07898508012294769,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8424479365348816,
+      "step": 195
+    },
+    {
+      "completion_length": 119.78125,
+      "epoch": 0.2613333333333333,
+      "grad_norm": 1.5365181064230193,
+      "kl": 0.041748046875,
+      "learning_rate": 8.693333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.829687476158142,
+      "reward_std": 0.07563021779060364,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8296874761581421,
+      "step": 196
+    },
+    {
+      "completion_length": 117.15625,
+      "epoch": 0.26266666666666666,
+      "grad_norm": 0.8700291149363001,
+      "kl": 0.023681640625,
+      "learning_rate": 8.686666666666666e-07,
+      "loss": 0.0009,
+      "reward": 1.6494791507720947,
+      "reward_std": 0.09973391890525818,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6651041507720947,
+      "step": 197
+    },
+    {
+      "completion_length": 127.15625,
+      "epoch": 0.264,
+      "grad_norm": 2.059530125350979,
+      "kl": 0.0458984375,
+      "learning_rate": 8.68e-07,
+      "loss": 0.0018,
+      "reward": 1.7802083492279053,
+      "reward_std": 0.1288667619228363,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7802082300186157,
+      "step": 198
+    },
+    {
+      "completion_length": 141.625,
+      "epoch": 0.2653333333333333,
+      "grad_norm": 8.218500422159094,
+      "kl": 0.039794921875,
+      "learning_rate": 8.673333333333332e-07,
+      "loss": 0.0016,
+      "reward": 1.774181604385376,
+      "reward_std": 0.16786371171474457,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7898065447807312,
+      "step": 199
+    },
+    {
+      "completion_length": 122.671875,
+      "epoch": 0.26666666666666666,
+      "grad_norm": 2.0168025158879646,
+      "kl": 0.0279541015625,
+      "learning_rate": 8.666666666666667e-07,
+      "loss": 0.0011,
+      "reward": 1.7615327835083008,
+      "reward_std": 0.22746162116527557,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.761532723903656,
+      "step": 200
+    },
+    {
+      "completion_length": 130.234375,
+      "epoch": 0.268,
+      "grad_norm": 1.2887792733479304,
+      "kl": 0.046142578125,
+      "learning_rate": 8.659999999999999e-07,
+      "loss": 0.0018,
+      "reward": 1.8062500953674316,
+      "reward_std": 0.055876053869724274,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8062499761581421,
+      "step": 201
+    },
+    {
+      "completion_length": 121.71875,
+      "epoch": 0.2693333333333333,
+      "grad_norm": 2.592836608826649,
+      "kl": 0.031982421875,
+      "learning_rate": 8.653333333333333e-07,
+      "loss": 0.0013,
+      "reward": 1.7197916507720947,
+      "reward_std": 0.11117921769618988,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7354166507720947,
+      "step": 202
+    },
+    {
+      "completion_length": 140.03125,
+      "epoch": 0.27066666666666667,
+      "grad_norm": 4.380490898620117,
+      "kl": 0.037841796875,
+      "learning_rate": 8.646666666666667e-07,
+      "loss": 0.0015,
+      "reward": 1.6531250476837158,
+      "reward_std": 0.2043299376964569,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.7312500476837158,
+      "step": 203
+    },
+    {
+      "completion_length": 128.84375,
+      "epoch": 0.272,
+      "grad_norm": 2.5304919301533064,
+      "kl": 0.05078125,
+      "learning_rate": 8.639999999999999e-07,
+      "loss": 0.002,
+      "reward": 1.8958333730697632,
+      "reward_std": 0.0624999925494194,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9114583730697632,
+      "step": 204
+    },
+    {
+      "completion_length": 136.5625,
+      "epoch": 0.2733333333333333,
+      "grad_norm": 1.4320761776517963,
+      "kl": 0.048095703125,
+      "learning_rate": 8.633333333333333e-07,
+      "loss": 0.0019,
+      "reward": 1.7018229961395264,
+      "reward_std": 0.06611742824316025,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7174479365348816,
+      "step": 205
+    },
+    {
+      "completion_length": 131.703125,
+      "epoch": 0.27466666666666667,
+      "grad_norm": 13.180789742736312,
+      "kl": 0.042236328125,
+      "learning_rate": 8.626666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.8934895992279053,
+      "reward_std": 0.11015230417251587,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.89348965883255,
+      "step": 206
+    },
+    {
+      "completion_length": 128.5,
+      "epoch": 0.276,
+      "grad_norm": 4.916213456579188,
+      "kl": 0.04443359375,
+      "learning_rate": 8.62e-07,
+      "loss": 0.0018,
+      "reward": 1.7408854961395264,
+      "reward_std": 0.17161604762077332,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7565104365348816,
+      "step": 207
+    },
+    {
+      "completion_length": 138.3125,
+      "epoch": 0.2773333333333333,
+      "grad_norm": 1.9898368198547185,
+      "kl": 0.048095703125,
+      "learning_rate": 8.613333333333332e-07,
+      "loss": 0.0019,
+      "reward": 1.8619792461395264,
+      "reward_std": 0.08352668583393097,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8619791269302368,
+      "step": 208
+    },
+    {
+      "completion_length": 150.0,
+      "epoch": 0.2786666666666667,
+      "grad_norm": 2.175119172752318,
+      "kl": 0.0262451171875,
+      "learning_rate": 8.606666666666667e-07,
+      "loss": 0.0011,
+      "reward": 1.6577380895614624,
+      "reward_std": 0.19530057907104492,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7202381491661072,
+      "step": 209
+    },
+    {
+      "completion_length": 124.25,
+      "epoch": 0.28,
+      "grad_norm": 1.6796859495731065,
+      "kl": 0.025146484375,
+      "learning_rate": 8.599999999999999e-07,
+      "loss": 0.001,
+      "reward": 1.75,
+      "reward_std": 0.19716878235340118,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.75,
+      "step": 210
+    },
+    {
+      "completion_length": 140.3125,
+      "epoch": 0.2813333333333333,
+      "grad_norm": 3.5245392559256277,
+      "kl": 0.06298828125,
+      "learning_rate": 8.593333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.7479166984558105,
+      "reward_std": 0.128628671169281,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7479166984558105,
+      "step": 211
+    },
+    {
+      "completion_length": 137.84375,
+      "epoch": 0.2826666666666667,
+      "grad_norm": 1.9972483019535872,
+      "kl": 0.037109375,
+      "learning_rate": 8.586666666666666e-07,
+      "loss": 0.0015,
+      "reward": 1.8151042461395264,
+      "reward_std": 0.0999799594283104,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8307291865348816,
+      "step": 212
+    },
+    {
+      "completion_length": 131.71875,
+      "epoch": 0.284,
+      "grad_norm": 1.317906108317647,
+      "kl": 0.049560546875,
+      "learning_rate": 8.58e-07,
+      "loss": 0.002,
+      "reward": 1.718360424041748,
+      "reward_std": 0.19960850477218628,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.749610424041748,
+      "step": 213
+    },
+    {
+      "completion_length": 141.421875,
+      "epoch": 0.2853333333333333,
+      "grad_norm": 1.4623958743555492,
+      "kl": 0.03662109375,
+      "learning_rate": 8.573333333333332e-07,
+      "loss": 0.0015,
+      "reward": 1.796875,
+      "reward_std": 0.11018073558807373,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.84375,
+      "step": 214
+    },
+    {
+      "completion_length": 130.140625,
+      "epoch": 0.2866666666666667,
+      "grad_norm": 2.7818082766168875,
+      "kl": 0.0498046875,
+      "learning_rate": 8.566666666666667e-07,
+      "loss": 0.002,
+      "reward": 1.6843750476837158,
+      "reward_std": 0.1884779930114746,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7000000476837158,
+      "step": 215
+    },
+    {
+      "completion_length": 115.46875,
+      "epoch": 0.288,
+      "grad_norm": 0.7504113241319471,
+      "kl": 0.0286865234375,
+      "learning_rate": 8.559999999999999e-07,
+      "loss": 0.0011,
+      "reward": 1.8932292461395264,
+      "reward_std": 0.046875,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9088541865348816,
+      "step": 216
+    },
+    {
+      "completion_length": 115.625,
+      "epoch": 0.28933333333333333,
+      "grad_norm": 23.869731208092045,
+      "kl": 0.06640625,
+      "learning_rate": 8.553333333333333e-07,
+      "loss": 0.0026,
+      "reward": 1.83984375,
+      "reward_std": 0.04389689117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8398438096046448,
+      "step": 217
+    },
+    {
+      "completion_length": 130.609375,
+      "epoch": 0.2906666666666667,
+      "grad_norm": 2.568914615779225,
+      "kl": 0.051025390625,
+      "learning_rate": 8.546666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.767968773841858,
+      "reward_std": 0.07675810903310776,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7679687738418579,
+      "step": 218
+    },
+    {
+      "completion_length": 139.28125,
+      "epoch": 0.292,
+      "grad_norm": 0.9435205822057867,
+      "kl": 0.034423828125,
+      "learning_rate": 8.539999999999999e-07,
+      "loss": 0.0014,
+      "reward": 1.7760417461395264,
+      "reward_std": 0.1848391890525818,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8229166865348816,
+      "step": 219
+    },
+    {
+      "completion_length": 136.953125,
+      "epoch": 0.29333333333333333,
+      "grad_norm": 2.3689867997672347,
+      "kl": 0.0537109375,
+      "learning_rate": 8.533333333333334e-07,
+      "loss": 0.0021,
+      "reward": 1.807031273841858,
+      "reward_std": 0.16633039712905884,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8382812738418579,
+      "step": 220
+    },
+    {
+      "completion_length": 136.765625,
+      "epoch": 0.2946666666666667,
+      "grad_norm": 1.5261803453399365,
+      "kl": 0.039794921875,
+      "learning_rate": 8.526666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.932031273841858,
+      "reward_std": 0.0859375,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9320312738418579,
+      "step": 221
+    },
+    {
+      "completion_length": 119.765625,
+      "epoch": 0.296,
+      "grad_norm": 2.647792095955694,
+      "kl": 0.055908203125,
+      "learning_rate": 8.52e-07,
+      "loss": 0.0022,
+      "reward": 1.8447916507720947,
+      "reward_std": 0.09785426408052444,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8604166507720947,
+      "step": 222
+    },
+    {
+      "completion_length": 120.671875,
+      "epoch": 0.29733333333333334,
+      "grad_norm": 1.7672095837481376,
+      "kl": 0.064453125,
+      "learning_rate": 8.513333333333333e-07,
+      "loss": 0.0026,
+      "reward": 1.8237723112106323,
+      "reward_std": 0.09598056226968765,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8393973112106323,
+      "step": 223
+    },
+    {
+      "completion_length": 130.21875,
+      "epoch": 0.2986666666666667,
+      "grad_norm": 2.0127732119103547,
+      "kl": 0.040283203125,
+      "learning_rate": 8.506666666666667e-07,
+      "loss": 0.0016,
+      "reward": 1.793229103088379,
+      "reward_std": 0.2225876748561859,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8088542222976685,
+      "step": 224
+    },
+    {
+      "completion_length": 125.71875,
+      "epoch": 0.3,
+      "grad_norm": 1.1409638004775335,
+      "kl": 0.05712890625,
+      "learning_rate": 8.499999999999999e-07,
+      "loss": 0.0023,
+      "reward": 1.8815104961395264,
+      "reward_std": 0.11243421584367752,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.9283853769302368,
+      "step": 225
+    },
+    {
+      "completion_length": 117.765625,
+      "epoch": 0.30133333333333334,
+      "grad_norm": 1.6334666220499197,
+      "kl": 0.0400390625,
+      "learning_rate": 8.493333333333334e-07,
+      "loss": 0.0016,
+      "reward": 1.8658854961395264,
+      "reward_std": 0.10182645171880722,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8815104365348816,
+      "step": 226
+    },
+    {
+      "completion_length": 128.078125,
+      "epoch": 0.30266666666666664,
+      "grad_norm": 0.9237649526800824,
+      "kl": 0.050048828125,
+      "learning_rate": 8.486666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.671875,
+      "reward_std": 0.16400586068630219,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.7499999403953552,
+      "step": 227
+    },
+    {
+      "completion_length": 122.421875,
+      "epoch": 0.304,
+      "grad_norm": 2.139844435843635,
+      "kl": 0.044189453125,
+      "learning_rate": 8.48e-07,
+      "loss": 0.0018,
+      "reward": 1.814843773841858,
+      "reward_std": 0.1104910671710968,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8304687738418579,
+      "step": 228
+    },
+    {
+      "completion_length": 117.546875,
+      "epoch": 0.30533333333333335,
+      "grad_norm": 0.6920622655004838,
+      "kl": 0.040771484375,
+      "learning_rate": 8.473333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.845312476158142,
+      "reward_std": 0.0031249995809048414,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8453124761581421,
+      "step": 229
+    },
+    {
+      "completion_length": 136.140625,
+      "epoch": 0.30666666666666664,
+      "grad_norm": 2.039938662764998,
+      "kl": 0.023193359375,
+      "learning_rate": 8.466666666666667e-07,
+      "loss": 0.0009,
+      "reward": 1.8312499523162842,
+      "reward_std": 0.2002945840358734,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8781249523162842,
+      "step": 230
+    },
+    {
+      "completion_length": 129.328125,
+      "epoch": 0.308,
+      "grad_norm": 4.693990620925548,
+      "kl": 0.037353515625,
+      "learning_rate": 8.459999999999999e-07,
+      "loss": 0.0015,
+      "reward": 1.759374976158142,
+      "reward_std": 0.2327350378036499,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7749999761581421,
+      "step": 231
+    },
+    {
+      "completion_length": 129.84375,
+      "epoch": 0.30933333333333335,
+      "grad_norm": 1.8194685164470241,
+      "kl": 0.039794921875,
+      "learning_rate": 8.453333333333334e-07,
+      "loss": 0.0016,
+      "reward": 1.6085937023162842,
+      "reward_std": 0.0928034633398056,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.608593761920929,
+      "step": 232
+    },
+    {
+      "completion_length": 131.09375,
+      "epoch": 0.31066666666666665,
+      "grad_norm": 1.7222793289712528,
+      "kl": 0.03759765625,
+      "learning_rate": 8.446666666666666e-07,
+      "loss": 0.0015,
+      "reward": 1.8229167461395264,
+      "reward_std": 0.07574943453073502,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8229166269302368,
+      "step": 233
+    },
+    {
+      "completion_length": 124.46875,
+      "epoch": 0.312,
+      "grad_norm": 6.81004796085756,
+      "kl": 0.05615234375,
+      "learning_rate": 8.439999999999999e-07,
+      "loss": 0.0022,
+      "reward": 1.779687523841858,
+      "reward_std": 0.18181249499320984,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7796875238418579,
+      "step": 234
+    },
+    {
+      "completion_length": 119.84375,
+      "epoch": 0.31333333333333335,
+      "grad_norm": 1.4122083975217838,
+      "kl": 0.037109375,
+      "learning_rate": 8.433333333333333e-07,
+      "loss": 0.0015,
+      "reward": 1.9114583730697632,
+      "reward_std": 0.0729166641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9114583730697632,
+      "step": 235
+    },
+    {
+      "completion_length": 118.6875,
+      "epoch": 0.31466666666666665,
+      "grad_norm": 19.07492867368131,
+      "kl": 0.0654296875,
+      "learning_rate": 8.426666666666666e-07,
+      "loss": 0.0026,
+      "reward": 1.8192708492279053,
+      "reward_std": 0.11500250548124313,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8348958492279053,
+      "step": 236
+    },
+    {
+      "completion_length": 120.546875,
+      "epoch": 0.316,
+      "grad_norm": 1.5164597998171836,
+      "kl": 0.048583984375,
+      "learning_rate": 8.419999999999999e-07,
+      "loss": 0.0019,
+      "reward": 1.6617188453674316,
+      "reward_std": 0.06490848958492279,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6617187857627869,
+      "step": 237
+    },
+    {
+      "completion_length": 119.734375,
+      "epoch": 0.31733333333333336,
+      "grad_norm": 3.4161754705964693,
+      "kl": 0.04541015625,
+      "learning_rate": 8.413333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.8307292461395264,
+      "reward_std": 0.1492721438407898,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8307291865348816,
+      "step": 238
+    },
+    {
+      "completion_length": 121.109375,
+      "epoch": 0.31866666666666665,
+      "grad_norm": 2.2666073452672886,
+      "kl": 0.039306640625,
+      "learning_rate": 8.406666666666667e-07,
+      "loss": 0.0016,
+      "reward": 1.767968773841858,
+      "reward_std": 0.1816287785768509,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7835937738418579,
+      "step": 239
+    },
+    {
+      "completion_length": 116.3125,
+      "epoch": 0.32,
+      "grad_norm": 1.692302574022033,
+      "kl": 0.068359375,
+      "learning_rate": 8.399999999999999e-07,
+      "loss": 0.0027,
+      "reward": 1.8603050708770752,
+      "reward_std": 0.022106792777776718,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.86030513048172,
+      "step": 240
+    },
+    {
+      "completion_length": 127.375,
+      "epoch": 0.32133333333333336,
+      "grad_norm": 2.061949850323971,
+      "kl": 0.05029296875,
+      "learning_rate": 8.393333333333334e-07,
+      "loss": 0.002,
+      "reward": 1.7638020515441895,
+      "reward_std": 0.10934494435787201,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.763802170753479,
+      "step": 241
+    },
+    {
+      "completion_length": 121.734375,
+      "epoch": 0.32266666666666666,
+      "grad_norm": 1.8391315713917284,
+      "kl": 0.03955078125,
+      "learning_rate": 8.386666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.9322917461395264,
+      "reward_std": 0.06021641939878464,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9322916865348816,
+      "step": 242
+    },
+    {
+      "completion_length": 127.171875,
+      "epoch": 0.324,
+      "grad_norm": 1.5659234536503581,
+      "kl": 0.02197265625,
+      "learning_rate": 8.38e-07,
+      "loss": 0.0009,
+      "reward": 1.6828124523162842,
+      "reward_std": 0.1888248175382614,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6984374523162842,
+      "step": 243
+    },
+    {
+      "completion_length": 136.96875,
+      "epoch": 0.3253333333333333,
+      "grad_norm": 3.3315888833012486,
+      "kl": 0.053955078125,
+      "learning_rate": 8.373333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.7989583015441895,
+      "reward_std": 0.20947103202342987,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8302083015441895,
+      "step": 244
+    },
+    {
+      "completion_length": 147.265625,
+      "epoch": 0.32666666666666666,
+      "grad_norm": 2.729350161405718,
+      "kl": 0.05712890625,
+      "learning_rate": 8.366666666666667e-07,
+      "loss": 0.0023,
+      "reward": 1.6135789155960083,
+      "reward_std": 0.1942390352487564,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.6604539155960083,
+      "step": 245
+    },
+    {
+      "completion_length": 126.21875,
+      "epoch": 0.328,
+      "grad_norm": 2.695617433346773,
+      "kl": 0.039306640625,
+      "learning_rate": 8.359999999999999e-07,
+      "loss": 0.0016,
+      "reward": 1.7408854961395264,
+      "reward_std": 0.15904733538627625,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7408854365348816,
+      "step": 246
+    },
+    {
+      "completion_length": 124.609375,
+      "epoch": 0.3293333333333333,
+      "grad_norm": 4.192992589346151,
+      "kl": 0.03271484375,
+      "learning_rate": 8.353333333333334e-07,
+      "loss": 0.0013,
+      "reward": 1.7917969226837158,
+      "reward_std": 0.10160593688488007,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7917969226837158,
+      "step": 247
+    },
+    {
+      "completion_length": 129.390625,
+      "epoch": 0.33066666666666666,
+      "grad_norm": 2.764299742745063,
+      "kl": 0.049072265625,
+      "learning_rate": 8.346666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.7976562976837158,
+      "reward_std": 0.11859256029129028,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7976562976837158,
+      "step": 248
+    },
+    {
+      "completion_length": 135.140625,
+      "epoch": 0.332,
+      "grad_norm": 1.857561286428319,
+      "kl": 0.04736328125,
+      "learning_rate": 8.34e-07,
+      "loss": 0.0019,
+      "reward": 1.7611979246139526,
+      "reward_std": 0.094932422041893,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7611979246139526,
+      "step": 249
+    },
+    {
+      "completion_length": 135.046875,
+      "epoch": 0.3333333333333333,
+      "grad_norm": 1.7998526691231371,
+      "kl": 0.06494140625,
+      "learning_rate": 8.333333333333333e-07,
+      "loss": 0.0026,
+      "reward": 1.853906273841858,
+      "reward_std": 0.05279193073511124,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8539062738418579,
+      "step": 250
+    },
+    {
+      "completion_length": 121.71875,
+      "epoch": 0.33466666666666667,
+      "grad_norm": 2.426055472372945,
+      "kl": 0.049072265625,
+      "learning_rate": 8.326666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.680208444595337,
+      "reward_std": 0.12926997244358063,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6802083253860474,
+      "step": 251
+    },
+    {
+      "completion_length": 129.765625,
+      "epoch": 0.336,
+      "grad_norm": 1.69306843594496,
+      "kl": 0.038330078125,
+      "learning_rate": 8.319999999999999e-07,
+      "loss": 0.0015,
+      "reward": 1.7369792461395264,
+      "reward_std": 0.1231372132897377,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7369791269302368,
+      "step": 252
+    },
+    {
+      "completion_length": 133.53125,
+      "epoch": 0.3373333333333333,
+      "grad_norm": 2.497929507627615,
+      "kl": 0.0517578125,
+      "learning_rate": 8.313333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.7179688215255737,
+      "reward_std": 0.1256067454814911,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7179688215255737,
+      "step": 253
+    },
+    {
+      "completion_length": 139.828125,
+      "epoch": 0.33866666666666667,
+      "grad_norm": 3.2464551857291584,
+      "kl": 0.061279296875,
+      "learning_rate": 8.306666666666666e-07,
+      "loss": 0.0025,
+      "reward": 1.6505208015441895,
+      "reward_std": 0.1632775366306305,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6505208015441895,
+      "step": 254
+    },
+    {
+      "completion_length": 141.15625,
+      "epoch": 0.34,
+      "grad_norm": 1.817097088227207,
+      "kl": 0.048828125,
+      "learning_rate": 8.299999999999999e-07,
+      "loss": 0.002,
+      "reward": 1.7442708015441895,
+      "reward_std": 0.10914000868797302,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7755208611488342,
+      "step": 255
+    },
+    {
+      "completion_length": 139.125,
+      "epoch": 0.3413333333333333,
+      "grad_norm": 1.4311943161545142,
+      "kl": 0.032958984375,
+      "learning_rate": 8.293333333333333e-07,
+      "loss": 0.0013,
+      "reward": 1.6966146230697632,
+      "reward_std": 0.10639689117670059,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7122395634651184,
+      "step": 256
+    },
+    {
+      "completion_length": 143.859375,
+      "epoch": 0.3426666666666667,
+      "grad_norm": 2.2978421282917867,
+      "kl": 0.04833984375,
+      "learning_rate": 8.286666666666666e-07,
+      "loss": 0.0019,
+      "reward": 1.5234375,
+      "reward_std": 0.16598501801490784,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.5390625,
+      "step": 257
+    },
+    {
+      "completion_length": 127.46875,
+      "epoch": 0.344,
+      "grad_norm": 1.6966332211765385,
+      "kl": 0.028564453125,
+      "learning_rate": 8.28e-07,
+      "loss": 0.0011,
+      "reward": 1.899999976158142,
+      "reward_std": 0.12868067622184753,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8999999761581421,
+      "step": 258
+    },
+    {
+      "completion_length": 135.234375,
+      "epoch": 0.3453333333333333,
+      "grad_norm": 1.8548693289723572,
+      "kl": 0.0458984375,
+      "learning_rate": 8.273333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.8128392696380615,
+      "reward_std": 0.14382171630859375,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8284642696380615,
+      "step": 259
+    },
+    {
+      "completion_length": 126.75,
+      "epoch": 0.3466666666666667,
+      "grad_norm": 1.9904951138282512,
+      "kl": 0.048828125,
+      "learning_rate": 8.266666666666667e-07,
+      "loss": 0.002,
+      "reward": 1.8450521230697632,
+      "reward_std": 0.10225088149309158,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8450521230697632,
+      "step": 260
+    },
+    {
+      "completion_length": 120.625,
+      "epoch": 0.348,
+      "grad_norm": 1.6188085876555354,
+      "kl": 0.05078125,
+      "learning_rate": 8.259999999999999e-07,
+      "loss": 0.002,
+      "reward": 1.77734375,
+      "reward_std": 0.08886225521564484,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7773436903953552,
+      "step": 261
+    },
+    {
+      "completion_length": 138.59375,
+      "epoch": 0.34933333333333333,
+      "grad_norm": 1.9744348861879564,
+      "kl": 0.072265625,
+      "learning_rate": 8.253333333333334e-07,
+      "loss": 0.0029,
+      "reward": 1.7361979484558105,
+      "reward_std": 0.06335469335317612,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7361979484558105,
+      "step": 262
+    },
+    {
+      "completion_length": 139.921875,
+      "epoch": 0.3506666666666667,
+      "grad_norm": 2.116089483051958,
+      "kl": 0.03857421875,
+      "learning_rate": 8.246666666666666e-07,
+      "loss": 0.0015,
+      "reward": 1.6809896230697632,
+      "reward_std": 0.2035001516342163,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7278646230697632,
+      "step": 263
+    },
+    {
+      "completion_length": 129.765625,
+      "epoch": 0.352,
+      "grad_norm": 3.2139665406756732,
+      "kl": 0.05859375,
+      "learning_rate": 8.24e-07,
+      "loss": 0.0023,
+      "reward": 1.8270833492279053,
+      "reward_std": 0.1987285614013672,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8270833492279053,
+      "step": 264
+    },
+    {
+      "completion_length": 128.6875,
+      "epoch": 0.35333333333333333,
+      "grad_norm": 1.9645490755236559,
+      "kl": 0.056640625,
+      "learning_rate": 8.233333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.8380208015441895,
+      "reward_std": 0.08317271620035172,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8380208015441895,
+      "step": 265
+    },
+    {
+      "completion_length": 133.8125,
+      "epoch": 0.3546666666666667,
+      "grad_norm": 2.1673447378538975,
+      "kl": 0.03271484375,
+      "learning_rate": 8.226666666666666e-07,
+      "loss": 0.0013,
+      "reward": 1.7890625,
+      "reward_std": 0.24545937776565552,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8359375,
+      "step": 266
+    },
+    {
+      "completion_length": 131.96875,
+      "epoch": 0.356,
+      "grad_norm": 2.1819538500879045,
+      "kl": 0.045166015625,
+      "learning_rate": 8.219999999999999e-07,
+      "loss": 0.0018,
+      "reward": 1.5596354007720947,
+      "reward_std": 0.06053442507982254,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.5596354007720947,
+      "step": 267
+    },
+    {
+      "completion_length": 124.421875,
+      "epoch": 0.35733333333333334,
+      "grad_norm": 3.644277014498408,
+      "kl": 0.049072265625,
+      "learning_rate": 8.213333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.6875,
+      "reward_std": 0.14478398859500885,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7187500596046448,
+      "step": 268
+    },
+    {
+      "completion_length": 116.0625,
+      "epoch": 0.3586666666666667,
+      "grad_norm": 1.4216320774011133,
+      "kl": 0.060546875,
+      "learning_rate": 8.206666666666666e-07,
+      "loss": 0.0024,
+      "reward": 1.865625023841858,
+      "reward_std": 0.12983438372612,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8812500238418579,
+      "step": 269
+    },
+    {
+      "completion_length": 129.703125,
+      "epoch": 0.36,
+      "grad_norm": 1.3606417627453886,
+      "kl": 0.053466796875,
+      "learning_rate": 8.199999999999999e-07,
+      "loss": 0.0021,
+      "reward": 1.8367187976837158,
+      "reward_std": 0.07796715199947357,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.852343738079071,
+      "step": 270
+    },
+    {
+      "completion_length": 118.3125,
+      "epoch": 0.36133333333333334,
+      "grad_norm": 3.920645817518093,
+      "kl": 0.048828125,
+      "learning_rate": 8.193333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.8307292461395264,
+      "reward_std": 0.057133615016937256,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8307291865348816,
+      "step": 271
+    },
+    {
+      "completion_length": 122.421875,
+      "epoch": 0.3626666666666667,
+      "grad_norm": 1.4926415440873986,
+      "kl": 0.061279296875,
+      "learning_rate": 8.186666666666666e-07,
+      "loss": 0.0024,
+      "reward": 1.7565104961395264,
+      "reward_std": 0.12444254755973816,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7721354365348816,
+      "step": 272
+    },
+    {
+      "completion_length": 118.984375,
+      "epoch": 0.364,
+      "grad_norm": 2.8261775800970903,
+      "kl": 0.060791015625,
+      "learning_rate": 8.179999999999999e-07,
+      "loss": 0.0024,
+      "reward": 1.7283854484558105,
+      "reward_std": 0.1064223125576973,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7283853888511658,
+      "step": 273
+    },
+    {
+      "completion_length": 132.84375,
+      "epoch": 0.36533333333333334,
+      "grad_norm": 1.4074885283580034,
+      "kl": 0.04443359375,
+      "learning_rate": 8.173333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.6484375,
+      "reward_std": 0.1614583283662796,
+      "rewards/format_reward": 0.890625,
+      "rewards/iou_reward": 0.7578125596046448,
+      "step": 274
+    },
+    {
+      "completion_length": 125.609375,
+      "epoch": 0.36666666666666664,
+      "grad_norm": 3.000345516636092,
+      "kl": 0.04052734375,
+      "learning_rate": 8.166666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.796875,
+      "reward_std": 0.10341878235340118,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.796875,
+      "step": 275
+    },
+    {
+      "completion_length": 124.28125,
+      "epoch": 0.368,
+      "grad_norm": 1.6792604388755485,
+      "kl": 0.048828125,
+      "learning_rate": 8.159999999999999e-07,
+      "loss": 0.0019,
+      "reward": 1.7083333730697632,
+      "reward_std": 0.12983438372612,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7708333134651184,
+      "step": 276
+    },
+    {
+      "completion_length": 127.15625,
+      "epoch": 0.36933333333333335,
+      "grad_norm": 1.830164539103774,
+      "kl": 0.04345703125,
+      "learning_rate": 8.153333333333334e-07,
+      "loss": 0.0017,
+      "reward": 1.76171875,
+      "reward_std": 0.08658864349126816,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.76171875,
+      "step": 277
+    },
+    {
+      "completion_length": 104.625,
+      "epoch": 0.37066666666666664,
+      "grad_norm": 1.6136312769696286,
+      "kl": 0.06298828125,
+      "learning_rate": 8.146666666666666e-07,
+      "loss": 0.0025,
+      "reward": 1.786718726158142,
+      "reward_std": 0.026090627536177635,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7867187857627869,
+      "step": 278
+    },
+    {
+      "completion_length": 116.21875,
+      "epoch": 0.372,
+      "grad_norm": 3.7338288231663865,
+      "kl": 0.047607421875,
+      "learning_rate": 8.14e-07,
+      "loss": 0.0019,
+      "reward": 1.7901041507720947,
+      "reward_std": 0.06848391890525818,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7901041507720947,
+      "step": 279
+    },
+    {
+      "completion_length": 122.296875,
+      "epoch": 0.37333333333333335,
+      "grad_norm": 1.1060376662177205,
+      "kl": 0.044189453125,
+      "learning_rate": 8.133333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.6927083730697632,
+      "reward_std": 0.07818284630775452,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7239583730697632,
+      "step": 280
+    },
+    {
+      "completion_length": 117.796875,
+      "epoch": 0.37466666666666665,
+      "grad_norm": 2.979921576009277,
+      "kl": 0.0888671875,
+      "learning_rate": 8.126666666666666e-07,
+      "loss": 0.0036,
+      "reward": 1.7549479007720947,
+      "reward_std": 0.09588229656219482,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7549479007720947,
+      "step": 281
+    },
+    {
+      "completion_length": 130.6875,
+      "epoch": 0.376,
+      "grad_norm": 2.16210532675928,
+      "kl": 0.052490234375,
+      "learning_rate": 8.12e-07,
+      "loss": 0.0021,
+      "reward": 1.7156250476837158,
+      "reward_std": 0.2948397397994995,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.746874988079071,
+      "step": 282
+    },
+    {
+      "completion_length": 112.40625,
+      "epoch": 0.37733333333333335,
+      "grad_norm": 1.7771597233863035,
+      "kl": 0.046142578125,
+      "learning_rate": 8.113333333333333e-07,
+      "loss": 0.0019,
+      "reward": 1.9135416746139526,
+      "reward_std": 0.041456207633018494,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9135416746139526,
+      "step": 283
+    },
+    {
+      "completion_length": 119.171875,
+      "epoch": 0.37866666666666665,
+      "grad_norm": 0.9393680463941673,
+      "kl": 0.076171875,
+      "learning_rate": 8.106666666666666e-07,
+      "loss": 0.003,
+      "reward": 1.9127604961395264,
+      "reward_std": 0.0703125,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9283854365348816,
+      "step": 284
+    },
+    {
+      "completion_length": 110.546875,
+      "epoch": 0.38,
+      "grad_norm": 2.1061206236381764,
+      "kl": 0.053955078125,
+      "learning_rate": 8.1e-07,
+      "loss": 0.0022,
+      "reward": 1.7723958492279053,
+      "reward_std": 0.078125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7723957896232605,
+      "step": 285
+    },
+    {
+      "completion_length": 116.84375,
+      "epoch": 0.38133333333333336,
+      "grad_norm": 1.4332621861002663,
+      "kl": 0.046142578125,
+      "learning_rate": 8.093333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.8026041984558105,
+      "reward_std": 0.16649705171585083,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.802604079246521,
+      "step": 286
+    },
+    {
+      "completion_length": 117.546875,
+      "epoch": 0.38266666666666665,
+      "grad_norm": 2.3943643885882517,
+      "kl": 0.07470703125,
+      "learning_rate": 8.086666666666666e-07,
+      "loss": 0.003,
+      "reward": 1.693750023841858,
+      "reward_std": 0.08602991700172424,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6937500238418579,
+      "step": 287
+    },
+    {
+      "completion_length": 114.84375,
+      "epoch": 0.384,
+      "grad_norm": 1.4609796501168817,
+      "kl": 0.0751953125,
+      "learning_rate": 8.08e-07,
+      "loss": 0.003,
+      "reward": 1.9427083730697632,
+      "reward_std": 0.03125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9427083730697632,
+      "step": 288
+    },
+    {
+      "completion_length": 120.296875,
+      "epoch": 0.38533333333333336,
+      "grad_norm": 1.8820513521365028,
+      "kl": 0.050537109375,
+      "learning_rate": 8.073333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.8364583253860474,
+      "reward_std": 0.14186251163482666,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8364583253860474,
+      "step": 289
+    },
+    {
+      "completion_length": 120.03125,
+      "epoch": 0.38666666666666666,
+      "grad_norm": 0.6538580001166251,
+      "kl": 0.04248046875,
+      "learning_rate": 8.066666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.6927083730697632,
+      "reward_std": 0.03125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6927083134651184,
+      "step": 290
+    },
+    {
+      "completion_length": 114.3125,
+      "epoch": 0.388,
+      "grad_norm": 2.106611217775037,
+      "kl": 0.0400390625,
+      "learning_rate": 8.06e-07,
+      "loss": 0.0016,
+      "reward": 1.84375,
+      "reward_std": 0.1555021107196808,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8437500596046448,
+      "step": 291
+    },
+    {
+      "completion_length": 130.609375,
+      "epoch": 0.3893333333333333,
+      "grad_norm": 1.5630565893335142,
+      "kl": 0.041748046875,
+      "learning_rate": 8.053333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.7956101894378662,
+      "reward_std": 0.06980380415916443,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8112351894378662,
+      "step": 292
+    },
+    {
+      "completion_length": 123.5625,
+      "epoch": 0.39066666666666666,
+      "grad_norm": 1.1258858931548539,
+      "kl": 0.043212890625,
+      "learning_rate": 8.046666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.9010417461395264,
+      "reward_std": 0.10341878235340118,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9010416865348816,
+      "step": 293
+    },
+    {
+      "completion_length": 118.53125,
+      "epoch": 0.392,
+      "grad_norm": 1.1910454055085855,
+      "kl": 0.041015625,
+      "learning_rate": 8.04e-07,
+      "loss": 0.0016,
+      "reward": 1.7947916984558105,
+      "reward_std": 0.0501958504319191,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7947916984558105,
+      "step": 294
+    },
+    {
+      "completion_length": 130.484375,
+      "epoch": 0.3933333333333333,
+      "grad_norm": 1.3246382632535636,
+      "kl": 0.036376953125,
+      "learning_rate": 8.033333333333333e-07,
+      "loss": 0.0015,
+      "reward": 1.847916603088379,
+      "reward_std": 0.10152509808540344,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8635416626930237,
+      "step": 295
+    },
+    {
+      "completion_length": 121.453125,
+      "epoch": 0.39466666666666667,
+      "grad_norm": 0.8746787480711644,
+      "kl": 0.03369140625,
+      "learning_rate": 8.026666666666667e-07,
+      "loss": 0.0013,
+      "reward": 1.921875,
+      "reward_std": 0.010416664183139801,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9218750596046448,
+      "step": 296
+    },
+    {
+      "completion_length": 115.953125,
+      "epoch": 0.396,
+      "grad_norm": 0.6135333294786949,
+      "kl": 0.049072265625,
+      "learning_rate": 8.02e-07,
+      "loss": 0.002,
+      "reward": 1.9635417461395264,
+      "reward_std": 0.03125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9635416269302368,
+      "step": 297
+    },
+    {
+      "completion_length": 136.5,
+      "epoch": 0.3973333333333333,
+      "grad_norm": 2.2305984079641434,
+      "kl": 0.052978515625,
+      "learning_rate": 8.013333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.8203125,
+      "reward_std": 0.09865829348564148,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8359375,
+      "step": 298
+    },
+    {
+      "completion_length": 118.140625,
+      "epoch": 0.39866666666666667,
+      "grad_norm": 5.523780083827392,
+      "kl": 0.05810546875,
+      "learning_rate": 8.006666666666666e-07,
+      "loss": 0.0023,
+      "reward": 1.840364694595337,
+      "reward_std": 0.1466934084892273,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8403645157814026,
+      "step": 299
+    },
+    {
+      "completion_length": 121.390625,
+      "epoch": 0.4,
+      "grad_norm": 1.5594514030380004,
+      "kl": 0.0390625,
+      "learning_rate": 8e-07,
+      "loss": 0.0016,
+      "reward": 1.8385417461395264,
+      "reward_std": 0.1090010553598404,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8541666865348816,
+      "step": 300
+    },
+    {
+      "completion_length": 128.0625,
+      "epoch": 0.4013333333333333,
+      "grad_norm": 1.2510035134956734,
+      "kl": 0.041259765625,
+      "learning_rate": 7.993333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.79296875,
+      "reward_std": 0.08035522699356079,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.80859375,
+      "step": 301
+    },
+    {
+      "completion_length": 120.6875,
+      "epoch": 0.4026666666666667,
+      "grad_norm": 3.350135763523772,
+      "kl": 0.037109375,
+      "learning_rate": 7.986666666666666e-07,
+      "loss": 0.0015,
+      "reward": 1.7434896230697632,
+      "reward_std": 0.15297186374664307,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7434896230697632,
+      "step": 302
+    },
+    {
+      "completion_length": 122.203125,
+      "epoch": 0.404,
+      "grad_norm": 1.682449016302755,
+      "kl": 0.045654296875,
+      "learning_rate": 7.98e-07,
+      "loss": 0.0018,
+      "reward": 1.7486854791641235,
+      "reward_std": 0.13323676586151123,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7643104791641235,
+      "step": 303
+    },
+    {
+      "completion_length": 134.984375,
+      "epoch": 0.4053333333333333,
+      "grad_norm": 2.063535960294822,
+      "kl": 0.06005859375,
+      "learning_rate": 7.973333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.7643229961395264,
+      "reward_std": 0.16365788877010345,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7955728769302368,
+      "step": 304
+    },
+    {
+      "completion_length": 125.0625,
+      "epoch": 0.4066666666666667,
+      "grad_norm": 7.92586447857799,
+      "kl": 0.059326171875,
+      "learning_rate": 7.966666666666666e-07,
+      "loss": 0.0024,
+      "reward": 1.8229167461395264,
+      "reward_std": 0.06733439117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8229166865348816,
+      "step": 305
+    },
+    {
+      "completion_length": 127.4375,
+      "epoch": 0.408,
+      "grad_norm": 2.3651357554337147,
+      "kl": 0.05078125,
+      "learning_rate": 7.96e-07,
+      "loss": 0.002,
+      "reward": 1.898695707321167,
+      "reward_std": 0.12055416405200958,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9143208265304565,
+      "step": 306
+    },
+    {
+      "completion_length": 115.578125,
+      "epoch": 0.4093333333333333,
+      "grad_norm": 2.1363920495652478,
+      "kl": 0.040283203125,
+      "learning_rate": 7.953333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.8541667461395264,
+      "reward_std": 0.13144585490226746,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8541666865348816,
+      "step": 307
+    },
+    {
+      "completion_length": 123.0625,
+      "epoch": 0.4106666666666667,
+      "grad_norm": 0.8845039643486029,
+      "kl": 0.0576171875,
+      "learning_rate": 7.946666666666666e-07,
+      "loss": 0.0023,
+      "reward": 1.8229166269302368,
+      "reward_std": 0.03975516930222511,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8229166865348816,
+      "step": 308
+    },
+    {
+      "completion_length": 120.015625,
+      "epoch": 0.412,
+      "grad_norm": 2.173392642732904,
+      "kl": 0.050048828125,
+      "learning_rate": 7.94e-07,
+      "loss": 0.002,
+      "reward": 1.8526785373687744,
+      "reward_std": 0.06925234198570251,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8526785969734192,
+      "step": 309
+    },
+    {
+      "completion_length": 118.0625,
+      "epoch": 0.41333333333333333,
+      "grad_norm": 2.509437702191442,
+      "kl": 0.046142578125,
+      "learning_rate": 7.933333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.90234375,
+      "reward_std": 0.06312988698482513,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.90234375,
+      "step": 310
+    },
+    {
+      "completion_length": 137.5625,
+      "epoch": 0.4146666666666667,
+      "grad_norm": 1.8874369164867577,
+      "kl": 0.043212890625,
+      "learning_rate": 7.926666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.8276041746139526,
+      "reward_std": 0.09408386051654816,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8432291746139526,
+      "step": 311
+    },
+    {
+      "completion_length": 118.265625,
+      "epoch": 0.416,
+      "grad_norm": 3.338526516563504,
+      "kl": 0.0625,
+      "learning_rate": 7.92e-07,
+      "loss": 0.0025,
+      "reward": 1.8227306604385376,
+      "reward_std": 0.10765070468187332,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8227306604385376,
+      "step": 312
+    },
+    {
+      "completion_length": 121.8125,
+      "epoch": 0.41733333333333333,
+      "grad_norm": 1.2463427886138025,
+      "kl": 0.0220947265625,
+      "learning_rate": 7.913333333333332e-07,
+      "loss": 0.0009,
+      "reward": 1.828125,
+      "reward_std": 0.13157323002815247,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.859375,
+      "step": 313
+    },
+    {
+      "completion_length": 133.5,
+      "epoch": 0.4186666666666667,
+      "grad_norm": 2.61759162978902,
+      "kl": 0.05419921875,
+      "learning_rate": 7.906666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.636458396911621,
+      "reward_std": 0.08414781838655472,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6364583373069763,
+      "step": 314
+    },
+    {
+      "completion_length": 110.21875,
+      "epoch": 0.42,
+      "grad_norm": 1.472243082396339,
+      "kl": 0.039794921875,
+      "learning_rate": 7.9e-07,
+      "loss": 0.0016,
+      "reward": 1.883333444595337,
+      "reward_std": 0.043217841535806656,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8833333253860474,
+      "step": 315
+    },
+    {
+      "completion_length": 120.65625,
+      "epoch": 0.42133333333333334,
+      "grad_norm": 1.2392931933833184,
+      "kl": 0.05419921875,
+      "learning_rate": 7.893333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.8885416984558105,
+      "reward_std": 0.006250001490116119,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8885416984558105,
+      "step": 316
+    },
+    {
+      "completion_length": 142.546875,
+      "epoch": 0.4226666666666667,
+      "grad_norm": 0.7693085748627336,
+      "kl": 0.0615234375,
+      "learning_rate": 7.886666666666666e-07,
+      "loss": 0.0025,
+      "reward": 1.780505895614624,
+      "reward_std": 0.13693168759346008,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.8586309552192688,
+      "step": 317
+    },
+    {
+      "completion_length": 121.4375,
+      "epoch": 0.424,
+      "grad_norm": 2.1282271992437916,
+      "kl": 0.05908203125,
+      "learning_rate": 7.88e-07,
+      "loss": 0.0024,
+      "reward": 1.8250744342803955,
+      "reward_std": 0.042090605944395065,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8250743746757507,
+      "step": 318
+    },
+    {
+      "completion_length": 126.25,
+      "epoch": 0.42533333333333334,
+      "grad_norm": 2.195509975409884,
+      "kl": 0.0673828125,
+      "learning_rate": 7.873333333333333e-07,
+      "loss": 0.0027,
+      "reward": 1.6986979246139526,
+      "reward_std": 0.10693143308162689,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6986979246139526,
+      "step": 319
+    },
+    {
+      "completion_length": 133.859375,
+      "epoch": 0.4266666666666667,
+      "grad_norm": 6.178079670110062,
+      "kl": 0.10986328125,
+      "learning_rate": 7.866666666666666e-07,
+      "loss": 0.0044,
+      "reward": 1.6920758485794067,
+      "reward_std": 0.1284269094467163,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7545759677886963,
+      "step": 320
+    },
+    {
+      "completion_length": 114.9375,
+      "epoch": 0.428,
+      "grad_norm": 1.32954656559574,
+      "kl": 0.038330078125,
+      "learning_rate": 7.86e-07,
+      "loss": 0.0015,
+      "reward": 1.8359375,
+      "reward_std": 0.11243987828493118,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8671875,
+      "step": 321
+    },
+    {
+      "completion_length": 108.46875,
+      "epoch": 0.42933333333333334,
+      "grad_norm": 1.099469748903943,
+      "kl": 0.06298828125,
+      "learning_rate": 7.853333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.86328125,
+      "reward_std": 0.018229160457849503,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.86328125,
+      "step": 322
+    },
+    {
+      "completion_length": 124.359375,
+      "epoch": 0.43066666666666664,
+      "grad_norm": 1.731220389241762,
+      "kl": 0.045654296875,
+      "learning_rate": 7.846666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.8958333730697632,
+      "reward_std": 0.0454794242978096,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8958332538604736,
+      "step": 323
+    },
+    {
+      "completion_length": 121.828125,
+      "epoch": 0.432,
+      "grad_norm": 2.2590197862861023,
+      "kl": 0.054931640625,
+      "learning_rate": 7.84e-07,
+      "loss": 0.0022,
+      "reward": 1.8606771230697632,
+      "reward_std": 0.13422296941280365,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8606770634651184,
+      "step": 324
+    },
+    {
+      "completion_length": 125.71875,
+      "epoch": 0.43333333333333335,
+      "grad_norm": 1.1658671335389368,
+      "kl": 0.06298828125,
+      "learning_rate": 7.833333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.8401042222976685,
+      "reward_std": 0.02812500298023224,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8401042222976685,
+      "step": 325
+    },
+    {
+      "completion_length": 107.953125,
+      "epoch": 0.43466666666666665,
+      "grad_norm": 1.2088747885563411,
+      "kl": 0.0517578125,
+      "learning_rate": 7.826666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.8515625,
+      "reward_std": 0.1037927195429802,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8515625,
+      "step": 326
+    },
+    {
+      "completion_length": 113.421875,
+      "epoch": 0.436,
+      "grad_norm": 2.70586807277335,
+      "kl": 0.032470703125,
+      "learning_rate": 7.82e-07,
+      "loss": 0.0013,
+      "reward": 1.8794642686843872,
+      "reward_std": 0.1342986822128296,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8794642686843872,
+      "step": 327
+    },
+    {
+      "completion_length": 125.078125,
+      "epoch": 0.43733333333333335,
+      "grad_norm": 1.1192342283554495,
+      "kl": 0.05126953125,
+      "learning_rate": 7.813333333333332e-07,
+      "loss": 0.002,
+      "reward": 1.7252604961395264,
+      "reward_std": 0.08153489977121353,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7408854365348816,
+      "step": 328
+    },
+    {
+      "completion_length": 130.96875,
+      "epoch": 0.43866666666666665,
+      "grad_norm": 1.5571377548661456,
+      "kl": 0.06640625,
+      "learning_rate": 7.806666666666666e-07,
+      "loss": 0.0027,
+      "reward": 1.8416666984558105,
+      "reward_std": 0.12841877341270447,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8572916388511658,
+      "step": 329
+    },
+    {
+      "completion_length": 132.921875,
+      "epoch": 0.44,
+      "grad_norm": 17.57668893013154,
+      "kl": 0.049560546875,
+      "learning_rate": 7.799999999999999e-07,
+      "loss": 0.002,
+      "reward": 1.8531250953674316,
+      "reward_std": 0.20066770911216736,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8687500357627869,
+      "step": 330
+    },
+    {
+      "completion_length": 129.890625,
+      "epoch": 0.44133333333333336,
+      "grad_norm": 2.6276447182777805,
+      "kl": 0.05029296875,
+      "learning_rate": 7.793333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.722395896911621,
+      "reward_std": 0.18775644898414612,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7536458373069763,
+      "step": 331
+    },
+    {
+      "completion_length": 131.328125,
+      "epoch": 0.44266666666666665,
+      "grad_norm": 3.8434900489899726,
+      "kl": 0.07080078125,
+      "learning_rate": 7.786666666666665e-07,
+      "loss": 0.0028,
+      "reward": 1.788802146911621,
+      "reward_std": 0.075499527156353,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7888020873069763,
+      "step": 332
+    },
+    {
+      "completion_length": 123.40625,
+      "epoch": 0.444,
+      "grad_norm": 1.9414870885386497,
+      "kl": 0.05419921875,
+      "learning_rate": 7.78e-07,
+      "loss": 0.0022,
+      "reward": 1.8307292461395264,
+      "reward_std": 0.16855303943157196,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8463541269302368,
+      "step": 333
+    },
+    {
+      "completion_length": 117.875,
+      "epoch": 0.44533333333333336,
+      "grad_norm": 0.8171754984804233,
+      "kl": 0.0277099609375,
+      "learning_rate": 7.773333333333333e-07,
+      "loss": 0.0011,
+      "reward": 1.9739583730697632,
+      "reward_std": 0.043278127908706665,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9739583134651184,
+      "step": 334
+    },
+    {
+      "completion_length": 127.359375,
+      "epoch": 0.44666666666666666,
+      "grad_norm": 1.6284567347842718,
+      "kl": 0.07275390625,
+      "learning_rate": 7.766666666666666e-07,
+      "loss": 0.0029,
+      "reward": 1.839322805404663,
+      "reward_std": 0.10781250149011612,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8393229246139526,
+      "step": 335
+    },
+    {
+      "completion_length": 120.203125,
+      "epoch": 0.448,
+      "grad_norm": 3.501553992537257,
+      "kl": 0.0634765625,
+      "learning_rate": 7.76e-07,
+      "loss": 0.0025,
+      "reward": 1.9031250476837158,
+      "reward_std": 0.0572093166410923,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9031250476837158,
+      "step": 336
+    },
+    {
+      "completion_length": 128.15625,
+      "epoch": 0.4493333333333333,
+      "grad_norm": 1.7034301603476314,
+      "kl": 0.035400390625,
+      "learning_rate": 7.753333333333333e-07,
+      "loss": 0.0014,
+      "reward": 1.8119791746139526,
+      "reward_std": 0.16943776607513428,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8276041746139526,
+      "step": 337
+    },
+    {
+      "completion_length": 127.578125,
+      "epoch": 0.45066666666666666,
+      "grad_norm": 9.563416218524878,
+      "kl": 0.06591796875,
+      "learning_rate": 7.746666666666666e-07,
+      "loss": 0.0026,
+      "reward": 1.7638020515441895,
+      "reward_std": 0.08243855834007263,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7638020515441895,
+      "step": 338
+    },
+    {
+      "completion_length": 113.96875,
+      "epoch": 0.452,
+      "grad_norm": 2.085405678360661,
+      "kl": 0.04638671875,
+      "learning_rate": 7.74e-07,
+      "loss": 0.0019,
+      "reward": 1.8463542461395264,
+      "reward_std": 0.1197916641831398,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8619791269302368,
+      "step": 339
+    },
+    {
+      "completion_length": 117.65625,
+      "epoch": 0.4533333333333333,
+      "grad_norm": 2.3392810575878005,
+      "kl": 0.07861328125,
+      "learning_rate": 7.733333333333333e-07,
+      "loss": 0.0031,
+      "reward": 1.8150670528411865,
+      "reward_std": 0.06748512387275696,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.815066933631897,
+      "step": 340
+    },
+    {
+      "completion_length": 111.171875,
+      "epoch": 0.45466666666666666,
+      "grad_norm": 2.447981480842365,
+      "kl": 0.0556640625,
+      "learning_rate": 7.726666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.853124976158142,
+      "reward_std": 0.10554219782352448,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8531249761581421,
+      "step": 341
+    },
+    {
+      "completion_length": 120.71875,
+      "epoch": 0.456,
+      "grad_norm": 2.1767940004174795,
+      "kl": 0.051513671875,
+      "learning_rate": 7.72e-07,
+      "loss": 0.0021,
+      "reward": 1.7291667461395264,
+      "reward_std": 0.038759566843509674,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7291666865348816,
+      "step": 342
+    },
+    {
+      "completion_length": 114.484375,
+      "epoch": 0.4573333333333333,
+      "grad_norm": 0.8325333925190097,
+      "kl": 0.0517578125,
+      "learning_rate": 7.713333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.9401042461395264,
+      "reward_std": 0.04510548710823059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9401041269302368,
+      "step": 343
+    },
+    {
+      "completion_length": 114.265625,
+      "epoch": 0.45866666666666667,
+      "grad_norm": 1.621476908974518,
+      "kl": 0.053955078125,
+      "learning_rate": 7.706666666666667e-07,
+      "loss": 0.0022,
+      "reward": 1.838281273841858,
+      "reward_std": 0.09508661925792694,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8382812738418579,
+      "step": 344
+    },
+    {
+      "completion_length": 116.171875,
+      "epoch": 0.46,
+      "grad_norm": 3.0495863838621307,
+      "kl": 0.0634765625,
+      "learning_rate": 7.699999999999999e-07,
+      "loss": 0.0025,
+      "reward": 1.640625,
+      "reward_std": 0.19449923932552338,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.640625,
+      "step": 345
+    },
+    {
+      "completion_length": 110.859375,
+      "epoch": 0.4613333333333333,
+      "grad_norm": 2.016950625771725,
+      "kl": 0.05224609375,
+      "learning_rate": 7.693333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.863541603088379,
+      "reward_std": 0.11391551792621613,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8635416626930237,
+      "step": 346
+    },
+    {
+      "completion_length": 115.515625,
+      "epoch": 0.46266666666666667,
+      "grad_norm": 1.6530894086239625,
+      "kl": 0.05517578125,
+      "learning_rate": 7.686666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.7838542461395264,
+      "reward_std": 0.015625,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7838542461395264,
+      "step": 347
+    },
+    {
+      "completion_length": 113.125,
+      "epoch": 0.464,
+      "grad_norm": 1.3389421264891914,
+      "kl": 0.03369140625,
+      "learning_rate": 7.68e-07,
+      "loss": 0.0014,
+      "reward": 1.8294271230697632,
+      "reward_std": 0.1119791641831398,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8450521230697632,
+      "step": 348
+    },
+    {
+      "completion_length": 116.296875,
+      "epoch": 0.4653333333333333,
+      "grad_norm": 1.3088818555607435,
+      "kl": 0.050537109375,
+      "learning_rate": 7.673333333333332e-07,
+      "loss": 0.002,
+      "reward": 1.8804688453674316,
+      "reward_std": 0.08951468020677567,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.9117187261581421,
+      "step": 349
+    },
+    {
+      "completion_length": 120.609375,
+      "epoch": 0.4666666666666667,
+      "grad_norm": 1.6903721355799273,
+      "kl": 0.048095703125,
+      "learning_rate": 7.666666666666667e-07,
+      "loss": 0.0019,
+      "reward": 1.7005208730697632,
+      "reward_std": 0.08295939117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7005207538604736,
+      "step": 350
+    },
+    {
+      "completion_length": 116.421875,
+      "epoch": 0.468,
+      "grad_norm": 2.7822627587639337,
+      "kl": 0.080078125,
+      "learning_rate": 7.66e-07,
+      "loss": 0.0032,
+      "reward": 1.7195684909820557,
+      "reward_std": 0.14735613763332367,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7195684909820557,
+      "step": 351
+    },
+    {
+      "completion_length": 133.578125,
+      "epoch": 0.4693333333333333,
+      "grad_norm": 1.5009816383332288,
+      "kl": 0.04931640625,
+      "learning_rate": 7.653333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.7046875953674316,
+      "reward_std": 0.10781536251306534,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7671875357627869,
+      "step": 352
+    },
+    {
+      "completion_length": 113.953125,
+      "epoch": 0.4706666666666667,
+      "grad_norm": 0.8345379518771593,
+      "kl": 0.044189453125,
+      "learning_rate": 7.646666666666667e-07,
+      "loss": 0.0018,
+      "reward": 1.9500000476837158,
+      "reward_std": 0.03750000149011612,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.949999988079071,
+      "step": 353
+    },
+    {
+      "completion_length": 116.40625,
+      "epoch": 0.472,
+      "grad_norm": 1.7710496720281033,
+      "kl": 0.0576171875,
+      "learning_rate": 7.64e-07,
+      "loss": 0.0023,
+      "reward": 1.8064732551574707,
+      "reward_std": 0.09986739605665207,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8064732551574707,
+      "step": 354
+    },
+    {
+      "completion_length": 119.671875,
+      "epoch": 0.47333333333333333,
+      "grad_norm": 1.564958597060998,
+      "kl": 0.06298828125,
+      "learning_rate": 7.633333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.8934895992279053,
+      "reward_std": 0.02076531946659088,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8934895992279053,
+      "step": 355
+    },
+    {
+      "completion_length": 114.546875,
+      "epoch": 0.4746666666666667,
+      "grad_norm": 1.4624503260486714,
+      "kl": 0.0810546875,
+      "learning_rate": 7.626666666666667e-07,
+      "loss": 0.0032,
+      "reward": 1.8483630418777466,
+      "reward_std": 0.11785713583230972,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8639880418777466,
+      "step": 356
+    },
+    {
+      "completion_length": 111.453125,
+      "epoch": 0.476,
+      "grad_norm": 1.4355745203212298,
+      "kl": 0.040771484375,
+      "learning_rate": 7.62e-07,
+      "loss": 0.0016,
+      "reward": 1.8666666746139526,
+      "reward_std": 0.006250001490116119,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8666666746139526,
+      "step": 357
+    },
+    {
+      "completion_length": 114.234375,
+      "epoch": 0.47733333333333333,
+      "grad_norm": 3.248535449966256,
+      "kl": 0.040771484375,
+      "learning_rate": 7.613333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.7940104007720947,
+      "reward_std": 0.15012797713279724,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7940104007720947,
+      "step": 358
+    },
+    {
+      "completion_length": 124.53125,
+      "epoch": 0.4786666666666667,
+      "grad_norm": 1.8749737519522047,
+      "kl": 0.052001953125,
+      "learning_rate": 7.606666666666667e-07,
+      "loss": 0.0021,
+      "reward": 1.7657551765441895,
+      "reward_std": 0.1256939172744751,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7657551765441895,
+      "step": 359
+    },
+    {
+      "completion_length": 117.03125,
+      "epoch": 0.48,
+      "grad_norm": 1.215252334374907,
+      "kl": 0.060791015625,
+      "learning_rate": 7.599999999999999e-07,
+      "loss": 0.0024,
+      "reward": 1.7336680889129639,
+      "reward_std": 0.0828268900513649,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7336682081222534,
+      "step": 360
+    },
+    {
+      "completion_length": 115.0625,
+      "epoch": 0.48133333333333334,
+      "grad_norm": 0.6021803999018295,
+      "kl": 0.02978515625,
+      "learning_rate": 7.593333333333333e-07,
+      "loss": 0.0012,
+      "reward": 1.8020833730697632,
+      "reward_std": 0.0625,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8020833134651184,
+      "step": 361
+    },
+    {
+      "completion_length": 110.03125,
+      "epoch": 0.4826666666666667,
+      "grad_norm": 3.166978818814014,
+      "kl": 0.034912109375,
+      "learning_rate": 7.586666666666666e-07,
+      "loss": 0.0014,
+      "reward": 1.921875,
+      "reward_std": 0.06851406395435333,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.921875,
+      "step": 362
+    },
+    {
+      "completion_length": 117.5,
+      "epoch": 0.484,
+      "grad_norm": 2.029291789094694,
+      "kl": 0.06787109375,
+      "learning_rate": 7.58e-07,
+      "loss": 0.0027,
+      "reward": 1.6867187023162842,
+      "reward_std": 0.1568058431148529,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7023438215255737,
+      "step": 363
+    },
+    {
+      "completion_length": 110.375,
+      "epoch": 0.48533333333333334,
+      "grad_norm": 1.556873588623045,
+      "kl": 0.037841796875,
+      "learning_rate": 7.573333333333332e-07,
+      "loss": 0.0015,
+      "reward": 1.8468749523162842,
+      "reward_std": 0.12983438372612,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.846875011920929,
+      "step": 364
+    },
+    {
+      "completion_length": 118.875,
+      "epoch": 0.4866666666666667,
+      "grad_norm": 3.435852351830393,
+      "kl": 0.059814453125,
+      "learning_rate": 7.566666666666667e-07,
+      "loss": 0.0024,
+      "reward": 1.767187476158142,
+      "reward_std": 0.15794843435287476,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7671874761581421,
+      "step": 365
+    },
+    {
+      "completion_length": 120.09375,
+      "epoch": 0.488,
+      "grad_norm": 0.9450212230885414,
+      "kl": 0.04931640625,
+      "learning_rate": 7.559999999999999e-07,
+      "loss": 0.002,
+      "reward": 1.8843005895614624,
+      "reward_std": 0.039787657558918,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8843005895614624,
+      "step": 366
+    },
+    {
+      "completion_length": 110.328125,
+      "epoch": 0.48933333333333334,
+      "grad_norm": 4.400916897082985,
+      "kl": 0.056396484375,
+      "learning_rate": 7.553333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.8692708015441895,
+      "reward_std": 0.04296480119228363,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8692708015441895,
+      "step": 367
+    },
+    {
+      "completion_length": 110.84375,
+      "epoch": 0.49066666666666664,
+      "grad_norm": 5.155486654445825,
+      "kl": 0.0927734375,
+      "learning_rate": 7.546666666666666e-07,
+      "loss": 0.0037,
+      "reward": 1.8549851179122925,
+      "reward_std": 0.0329921692609787,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8549851179122925,
+      "step": 368
+    },
+    {
+      "completion_length": 100.96875,
+      "epoch": 0.492,
+      "grad_norm": 1.4751107817240374,
+      "kl": 0.062255859375,
+      "learning_rate": 7.54e-07,
+      "loss": 0.0025,
+      "reward": 1.8828125,
+      "reward_std": 0.0753338634967804,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8828124403953552,
+      "step": 369
+    },
+    {
+      "completion_length": 113.109375,
+      "epoch": 0.49333333333333335,
+      "grad_norm": 2.9836342661077007,
+      "kl": 0.08056640625,
+      "learning_rate": 7.533333333333332e-07,
+      "loss": 0.0032,
+      "reward": 1.796875,
+      "reward_std": 0.02873235195875168,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.796875,
+      "step": 370
+    },
+    {
+      "completion_length": 125.625,
+      "epoch": 0.49466666666666664,
+      "grad_norm": 3.7252433466863097,
+      "kl": 0.08984375,
+      "learning_rate": 7.526666666666667e-07,
+      "loss": 0.0036,
+      "reward": 1.7630208730697632,
+      "reward_std": 0.06590189039707184,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7630208134651184,
+      "step": 371
+    },
+    {
+      "completion_length": 126.859375,
+      "epoch": 0.496,
+      "grad_norm": 1.6334970848873587,
+      "kl": 0.0576171875,
+      "learning_rate": 7.52e-07,
+      "loss": 0.0023,
+      "reward": 1.742708444595337,
+      "reward_std": 0.1090010553598404,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.8052083253860474,
+      "step": 372
+    },
+    {
+      "completion_length": 119.625,
+      "epoch": 0.49733333333333335,
+      "grad_norm": 2.053111533488906,
+      "kl": 0.054931640625,
+      "learning_rate": 7.513333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.7791666984558105,
+      "reward_std": 0.1796424388885498,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8104166984558105,
+      "step": 373
+    },
+    {
+      "completion_length": 128.15625,
+      "epoch": 0.49866666666666665,
+      "grad_norm": 2.231997688417399,
+      "kl": 0.0517578125,
+      "learning_rate": 7.506666666666667e-07,
+      "loss": 0.0021,
+      "reward": 1.5656249523162842,
+      "reward_std": 0.11544691026210785,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.581250011920929,
+      "step": 374
+    },
+    {
+      "completion_length": 127.375,
+      "epoch": 0.5,
+      "grad_norm": 0.958724682621826,
+      "kl": 0.032958984375,
+      "learning_rate": 7.5e-07,
+      "loss": 0.0013,
+      "reward": 1.8971354961395264,
+      "reward_std": 0.1015625,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9127604365348816,
+      "step": 375
+    },
+    {
+      "completion_length": 120.4375,
+      "epoch": 0.5013333333333333,
+      "grad_norm": 2.2932413104115845,
+      "kl": 0.0546875,
+      "learning_rate": 7.493333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.792708396911621,
+      "reward_std": 0.1418856680393219,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8239583969116211,
+      "step": 376
+    },
+    {
+      "completion_length": 116.765625,
+      "epoch": 0.5026666666666667,
+      "grad_norm": 1.1564733790319668,
+      "kl": 0.03173828125,
+      "learning_rate": 7.486666666666666e-07,
+      "loss": 0.0013,
+      "reward": 1.8567708730697632,
+      "reward_std": 0.1332494616508484,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8723958730697632,
+      "step": 377
+    },
+    {
+      "completion_length": 119.390625,
+      "epoch": 0.504,
+      "grad_norm": 6.356811966212154,
+      "kl": 0.058349609375,
+      "learning_rate": 7.48e-07,
+      "loss": 0.0023,
+      "reward": 1.8294271230697632,
+      "reward_std": 0.16807948052883148,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8450521230697632,
+      "step": 378
+    },
+    {
+      "completion_length": 123.953125,
+      "epoch": 0.5053333333333333,
+      "grad_norm": 1.7731956073186188,
+      "kl": 0.037109375,
+      "learning_rate": 7.473333333333332e-07,
+      "loss": 0.0015,
+      "reward": 1.8463542461395264,
+      "reward_std": 0.10379272699356079,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8463541269302368,
+      "step": 379
+    },
+    {
+      "completion_length": 108.34375,
+      "epoch": 0.5066666666666667,
+      "grad_norm": 6.165979671344845,
+      "kl": 0.07568359375,
+      "learning_rate": 7.466666666666667e-07,
+      "loss": 0.003,
+      "reward": 1.809114694595337,
+      "reward_std": 0.14302657544612885,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8247395753860474,
+      "step": 380
+    },
+    {
+      "completion_length": 124.921875,
+      "epoch": 0.508,
+      "grad_norm": 3.4301209598953166,
+      "kl": 0.041259765625,
+      "learning_rate": 7.459999999999999e-07,
+      "loss": 0.0016,
+      "reward": 1.7755208015441895,
+      "reward_std": 0.1404900848865509,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.8380208015441895,
+      "step": 381
+    },
+    {
+      "completion_length": 113.359375,
+      "epoch": 0.5093333333333333,
+      "grad_norm": 1.413926241833073,
+      "kl": 0.0751953125,
+      "learning_rate": 7.453333333333333e-07,
+      "loss": 0.003,
+      "reward": 1.8588541746139526,
+      "reward_std": 0.0677083358168602,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8588541746139526,
+      "step": 382
+    },
+    {
+      "completion_length": 121.96875,
+      "epoch": 0.5106666666666667,
+      "grad_norm": 1.9266500023205613,
+      "kl": 0.0703125,
+      "learning_rate": 7.446666666666666e-07,
+      "loss": 0.0028,
+      "reward": 1.839583396911621,
+      "reward_std": 0.07452812790870667,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8395832777023315,
+      "step": 383
+    },
+    {
+      "completion_length": 129.484375,
+      "epoch": 0.512,
+      "grad_norm": 0.6977162925180729,
+      "kl": 0.048828125,
+      "learning_rate": 7.44e-07,
+      "loss": 0.0019,
+      "reward": 1.860863208770752,
+      "reward_std": 0.07462453842163086,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8764881491661072,
+      "step": 384
+    },
+    {
+      "completion_length": 127.0625,
+      "epoch": 0.5133333333333333,
+      "grad_norm": 2.374420524467126,
+      "kl": 0.0771484375,
+      "learning_rate": 7.433333333333332e-07,
+      "loss": 0.0031,
+      "reward": 1.6744420528411865,
+      "reward_std": 0.1024916023015976,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.690066933631897,
+      "step": 385
+    },
+    {
+      "completion_length": 130.921875,
+      "epoch": 0.5146666666666667,
+      "grad_norm": 0.7649049272173901,
+      "kl": 0.0458984375,
+      "learning_rate": 7.426666666666667e-07,
+      "loss": 0.0018,
+      "reward": 1.75,
+      "reward_std": 0.125,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.78125,
+      "step": 386
+    },
+    {
+      "completion_length": 122.09375,
+      "epoch": 0.516,
+      "grad_norm": 1.5033291828846953,
+      "kl": 0.06201171875,
+      "learning_rate": 7.42e-07,
+      "loss": 0.0025,
+      "reward": 1.8033854961395264,
+      "reward_std": 0.11731092631816864,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8033854365348816,
+      "step": 387
+    },
+    {
+      "completion_length": 122.1875,
+      "epoch": 0.5173333333333333,
+      "grad_norm": 1.637001575904124,
+      "kl": 0.056396484375,
+      "learning_rate": 7.413333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.7979166507720947,
+      "reward_std": 0.09135933220386505,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7979167103767395,
+      "step": 388
+    },
+    {
+      "completion_length": 113.71875,
+      "epoch": 0.5186666666666667,
+      "grad_norm": 1.0318742164170882,
+      "kl": 0.038330078125,
+      "learning_rate": 7.406666666666667e-07,
+      "loss": 0.0015,
+      "reward": 1.935156226158142,
+      "reward_std": 0.07202189415693283,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9351562261581421,
+      "step": 389
+    },
+    {
+      "completion_length": 119.515625,
+      "epoch": 0.52,
+      "grad_norm": 1.075699326004039,
+      "kl": 0.04931640625,
+      "learning_rate": 7.4e-07,
+      "loss": 0.002,
+      "reward": 1.8229166269302368,
+      "reward_std": 0.054103441536426544,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8229166269302368,
+      "step": 390
+    },
+    {
+      "completion_length": 121.765625,
+      "epoch": 0.5213333333333333,
+      "grad_norm": 2.23307241208901,
+      "kl": 0.040283203125,
+      "learning_rate": 7.393333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.7410714626312256,
+      "reward_std": 0.0386904776096344,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7410714626312256,
+      "step": 391
+    },
+    {
+      "completion_length": 131.53125,
+      "epoch": 0.5226666666666666,
+      "grad_norm": 1.2712669258683695,
+      "kl": 0.052490234375,
+      "learning_rate": 7.386666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.7367607355117798,
+      "reward_std": 0.2066117823123932,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.8148857355117798,
+      "step": 392
+    },
+    {
+      "completion_length": 114.890625,
+      "epoch": 0.524,
+      "grad_norm": 7.2264939552762835,
+      "kl": 0.07080078125,
+      "learning_rate": 7.38e-07,
+      "loss": 0.0028,
+      "reward": 1.7098958492279053,
+      "reward_std": 0.09013683348894119,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7255208492279053,
+      "step": 393
+    },
+    {
+      "completion_length": 119.5625,
+      "epoch": 0.5253333333333333,
+      "grad_norm": 0.6978336098036734,
+      "kl": 0.036376953125,
+      "learning_rate": 7.373333333333332e-07,
+      "loss": 0.0015,
+      "reward": 1.8072917461395264,
+      "reward_std": 0.09375,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8229166865348816,
+      "step": 394
+    },
+    {
+      "completion_length": 125.21875,
+      "epoch": 0.5266666666666666,
+      "grad_norm": 5.055730219703559,
+      "kl": 0.0439453125,
+      "learning_rate": 7.366666666666667e-07,
+      "loss": 0.0018,
+      "reward": 1.6776041984558105,
+      "reward_std": 0.25787922739982605,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6932291388511658,
+      "step": 395
+    },
+    {
+      "completion_length": 120.734375,
+      "epoch": 0.528,
+      "grad_norm": 1.6286416874504053,
+      "kl": 0.051513671875,
+      "learning_rate": 7.359999999999999e-07,
+      "loss": 0.0021,
+      "reward": 1.664434552192688,
+      "reward_std": 0.05056828632950783,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.664434552192688,
+      "step": 396
+    },
+    {
+      "completion_length": 124.375,
+      "epoch": 0.5293333333333333,
+      "grad_norm": 1.8764954629918336,
+      "kl": 0.0458984375,
+      "learning_rate": 7.353333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.817968726158142,
+      "reward_std": 0.11769826710224152,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8179687261581421,
+      "step": 397
+    },
+    {
+      "completion_length": 128.1875,
+      "epoch": 0.5306666666666666,
+      "grad_norm": 2.021461399538331,
+      "kl": 0.05126953125,
+      "learning_rate": 7.346666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.7911458015441895,
+      "reward_std": 0.11515793949365616,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8067708015441895,
+      "step": 398
+    },
+    {
+      "completion_length": 128.609375,
+      "epoch": 0.532,
+      "grad_norm": 6.124564756377563,
+      "kl": 0.050537109375,
+      "learning_rate": 7.34e-07,
+      "loss": 0.002,
+      "reward": 1.890625,
+      "reward_std": 0.12425211817026138,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.890625,
+      "step": 399
+    },
+    {
+      "completion_length": 130.71875,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.579447543329729,
+      "kl": 0.04638671875,
+      "learning_rate": 7.333333333333332e-07,
+      "loss": 0.0019,
+      "reward": 1.82421875,
+      "reward_std": 0.16644902527332306,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.87109375,
+      "step": 400
+    },
+    {
+      "completion_length": 125.328125,
+      "epoch": 0.5346666666666666,
+      "grad_norm": 2.473810311102739,
+      "kl": 0.053466796875,
+      "learning_rate": 7.326666666666667e-07,
+      "loss": 0.0021,
+      "reward": 1.7546875476837158,
+      "reward_std": 0.18950852751731873,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.785937488079071,
+      "step": 401
+    },
+    {
+      "completion_length": 128.9375,
+      "epoch": 0.536,
+      "grad_norm": 2.1704330697643344,
+      "kl": 0.05517578125,
+      "learning_rate": 7.319999999999999e-07,
+      "loss": 0.0022,
+      "reward": 1.8497395515441895,
+      "reward_std": 0.16614583134651184,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8653646111488342,
+      "step": 402
+    },
+    {
+      "completion_length": 120.015625,
+      "epoch": 0.5373333333333333,
+      "grad_norm": 1.5544044437487252,
+      "kl": 0.06787109375,
+      "learning_rate": 7.313333333333333e-07,
+      "loss": 0.0027,
+      "reward": 1.9349702596664429,
+      "reward_std": 0.023381110280752182,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9349702596664429,
+      "step": 403
+    },
+    {
+      "completion_length": 120.046875,
+      "epoch": 0.5386666666666666,
+      "grad_norm": 1.4158247494448,
+      "kl": 0.059814453125,
+      "learning_rate": 7.306666666666666e-07,
+      "loss": 0.0024,
+      "reward": 1.8635417222976685,
+      "reward_std": 0.1295560747385025,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8791666626930237,
+      "step": 404
+    },
+    {
+      "completion_length": 123.125,
+      "epoch": 0.54,
+      "grad_norm": 1.3816421675740187,
+      "kl": 0.056640625,
+      "learning_rate": 7.3e-07,
+      "loss": 0.0023,
+      "reward": 1.8510416746139526,
+      "reward_std": 0.10698094218969345,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8822916746139526,
+      "step": 405
+    },
+    {
+      "completion_length": 116.59375,
+      "epoch": 0.5413333333333333,
+      "grad_norm": 6.524977840160972,
+      "kl": 0.09326171875,
+      "learning_rate": 7.293333333333332e-07,
+      "loss": 0.0037,
+      "reward": 1.770052194595337,
+      "reward_std": 0.12931355834007263,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7700520753860474,
+      "step": 406
+    },
+    {
+      "completion_length": 124.984375,
+      "epoch": 0.5426666666666666,
+      "grad_norm": 2.2167394760671217,
+      "kl": 0.0615234375,
+      "learning_rate": 7.286666666666666e-07,
+      "loss": 0.0025,
+      "reward": 1.8442708253860474,
+      "reward_std": 0.0451284721493721,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8442708253860474,
+      "step": 407
+    },
+    {
+      "completion_length": 106.984375,
+      "epoch": 0.544,
+      "grad_norm": 1.4790580159794722,
+      "kl": 0.030517578125,
+      "learning_rate": 7.28e-07,
+      "loss": 0.0012,
+      "reward": 1.8828125,
+      "reward_std": 0.109375,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8828125,
+      "step": 408
+    },
+    {
+      "completion_length": 126.6875,
+      "epoch": 0.5453333333333333,
+      "grad_norm": 1.3016896287717123,
+      "kl": 0.049072265625,
+      "learning_rate": 7.273333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.8997395038604736,
+      "reward_std": 0.05908758565783501,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8997395634651184,
+      "step": 409
+    },
+    {
+      "completion_length": 123.734375,
+      "epoch": 0.5466666666666666,
+      "grad_norm": 1.7284693212680504,
+      "kl": 0.08203125,
+      "learning_rate": 7.266666666666667e-07,
+      "loss": 0.0033,
+      "reward": 1.7651041746139526,
+      "reward_std": 0.1318429410457611,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7651042342185974,
+      "step": 410
+    },
+    {
+      "completion_length": 129.890625,
+      "epoch": 0.548,
+      "grad_norm": 1.5080134192643393,
+      "kl": 0.061279296875,
+      "learning_rate": 7.259999999999999e-07,
+      "loss": 0.0024,
+      "reward": 1.7703125476837158,
+      "reward_std": 0.05637823045253754,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7859375476837158,
+      "step": 411
+    },
+    {
+      "completion_length": 121.84375,
+      "epoch": 0.5493333333333333,
+      "grad_norm": 2.9724642267311605,
+      "kl": 0.061279296875,
+      "learning_rate": 7.253333333333334e-07,
+      "loss": 0.0025,
+      "reward": 1.81640625,
+      "reward_std": 0.0692700743675232,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.81640625,
+      "step": 412
+    },
+    {
+      "completion_length": 120.34375,
+      "epoch": 0.5506666666666666,
+      "grad_norm": 1.452300175225606,
+      "kl": 0.043212890625,
+      "learning_rate": 7.246666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.8958333730697632,
+      "reward_std": 0.12554723024368286,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8958333730697632,
+      "step": 413
+    },
+    {
+      "completion_length": 118.640625,
+      "epoch": 0.552,
+      "grad_norm": 7.714206247109425,
+      "kl": 0.03759765625,
+      "learning_rate": 7.24e-07,
+      "loss": 0.0015,
+      "reward": 1.8489583730697632,
+      "reward_std": 0.11544691026210785,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8489583134651184,
+      "step": 414
+    },
+    {
+      "completion_length": 121.15625,
+      "epoch": 0.5533333333333333,
+      "grad_norm": 2.5308983188243377,
+      "kl": 0.034912109375,
+      "learning_rate": 7.233333333333333e-07,
+      "loss": 0.0014,
+      "reward": 1.881250023841858,
+      "reward_std": 0.13466878235340118,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8968750238418579,
+      "step": 415
+    },
+    {
+      "completion_length": 113.484375,
+      "epoch": 0.5546666666666666,
+      "grad_norm": 8.13718642775787,
+      "kl": 0.08642578125,
+      "learning_rate": 7.226666666666667e-07,
+      "loss": 0.0034,
+      "reward": 1.7630208730697632,
+      "reward_std": 0.1619298756122589,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7786458134651184,
+      "step": 416
+    },
+    {
+      "completion_length": 123.46875,
+      "epoch": 0.556,
+      "grad_norm": 2.0526712666686935,
+      "kl": 0.0712890625,
+      "learning_rate": 7.219999999999999e-07,
+      "loss": 0.0029,
+      "reward": 1.8193824291229248,
+      "reward_std": 0.11160522699356079,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8193824291229248,
+      "step": 417
+    },
+    {
+      "completion_length": 119.421875,
+      "epoch": 0.5573333333333333,
+      "grad_norm": 2.5593200766288082,
+      "kl": 0.0732421875,
+      "learning_rate": 7.213333333333334e-07,
+      "loss": 0.0029,
+      "reward": 1.91015625,
+      "reward_std": 0.09796562790870667,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9101563096046448,
+      "step": 418
+    },
+    {
+      "completion_length": 137.875,
+      "epoch": 0.5586666666666666,
+      "grad_norm": 1.6480238681796926,
+      "kl": 0.052001953125,
+      "learning_rate": 7.206666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.5874255895614624,
+      "reward_std": 0.07531489431858063,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.6499256491661072,
+      "step": 419
+    },
+    {
+      "completion_length": 124.375,
+      "epoch": 0.56,
+      "grad_norm": 2.9863682319633122,
+      "kl": 0.0439453125,
+      "learning_rate": 7.2e-07,
+      "loss": 0.0018,
+      "reward": 1.809337854385376,
+      "reward_std": 0.2171562761068344,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.809337854385376,
+      "step": 420
+    },
+    {
+      "completion_length": 129.421875,
+      "epoch": 0.5613333333333334,
+      "grad_norm": 1.0033637145536065,
+      "kl": 0.035400390625,
+      "learning_rate": 7.193333333333333e-07,
+      "loss": 0.0014,
+      "reward": 1.7927827835083008,
+      "reward_std": 0.0647321417927742,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8396577835083008,
+      "step": 421
+    },
+    {
+      "completion_length": 128.0,
+      "epoch": 0.5626666666666666,
+      "grad_norm": 5.270217229016233,
+      "kl": 0.07275390625,
+      "learning_rate": 7.186666666666667e-07,
+      "loss": 0.0029,
+      "reward": 1.8728423118591309,
+      "reward_std": 0.03319663554430008,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8728423118591309,
+      "step": 422
+    },
+    {
+      "completion_length": 120.34375,
+      "epoch": 0.564,
+      "grad_norm": 2.5078123008015267,
+      "kl": 0.06103515625,
+      "learning_rate": 7.179999999999999e-07,
+      "loss": 0.0024,
+      "reward": 1.7330729961395264,
+      "reward_std": 0.03245859593153,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7330729365348816,
+      "step": 423
+    },
+    {
+      "completion_length": 125.5625,
+      "epoch": 0.5653333333333334,
+      "grad_norm": 4.631001326235269,
+      "kl": 0.07666015625,
+      "learning_rate": 7.173333333333333e-07,
+      "loss": 0.0031,
+      "reward": 1.7411458492279053,
+      "reward_std": 0.0845419317483902,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7567708492279053,
+      "step": 424
+    },
+    {
+      "completion_length": 127.375,
+      "epoch": 0.5666666666666667,
+      "grad_norm": 3.0778128264340814,
+      "kl": 0.07763671875,
+      "learning_rate": 7.166666666666667e-07,
+      "loss": 0.0031,
+      "reward": 1.7450520992279053,
+      "reward_std": 0.016266342252492905,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.8075520992279053,
+      "step": 425
+    },
+    {
+      "completion_length": 123.828125,
+      "epoch": 0.568,
+      "grad_norm": 4.6729547710973,
+      "kl": 0.08251953125,
+      "learning_rate": 7.159999999999999e-07,
+      "loss": 0.0033,
+      "reward": 1.705468773841858,
+      "reward_std": 0.17989182472229004,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7210937738418579,
+      "step": 426
+    },
+    {
+      "completion_length": 118.90625,
+      "epoch": 0.5693333333333334,
+      "grad_norm": 1.8653384209404587,
+      "kl": 0.06982421875,
+      "learning_rate": 7.153333333333334e-07,
+      "loss": 0.0028,
+      "reward": 1.8359375,
+      "reward_std": 0.1256677210330963,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8515625,
+      "step": 427
+    },
+    {
+      "completion_length": 129.90625,
+      "epoch": 0.5706666666666667,
+      "grad_norm": 1.934272639305947,
+      "kl": 0.06005859375,
+      "learning_rate": 7.146666666666666e-07,
+      "loss": 0.0024,
+      "reward": 1.6805059909820557,
+      "reward_std": 0.15799358487129211,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7273809313774109,
+      "step": 428
+    },
+    {
+      "completion_length": 111.890625,
+      "epoch": 0.572,
+      "grad_norm": 1.2797766088668379,
+      "kl": 0.048095703125,
+      "learning_rate": 7.14e-07,
+      "loss": 0.0019,
+      "reward": 1.845312476158142,
+      "reward_std": 0.022841880097985268,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8453124761581421,
+      "step": 429
+    },
+    {
+      "completion_length": 111.390625,
+      "epoch": 0.5733333333333334,
+      "grad_norm": 1.768682110566369,
+      "kl": 0.037109375,
+      "learning_rate": 7.133333333333333e-07,
+      "loss": 0.0015,
+      "reward": 1.9166667461395264,
+      "reward_std": 0.044589564204216,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9166666865348816,
+      "step": 430
+    },
+    {
+      "completion_length": 113.15625,
+      "epoch": 0.5746666666666667,
+      "grad_norm": 2.5300032648731725,
+      "kl": 0.06884765625,
+      "learning_rate": 7.126666666666667e-07,
+      "loss": 0.0028,
+      "reward": 1.8072917461395264,
+      "reward_std": 0.10232061892747879,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8072916865348816,
+      "step": 431
+    },
+    {
+      "completion_length": 110.203125,
+      "epoch": 0.576,
+      "grad_norm": 1.7746468123607277,
+      "kl": 0.08642578125,
+      "learning_rate": 7.119999999999999e-07,
+      "loss": 0.0035,
+      "reward": 1.8406250476837158,
+      "reward_std": 0.14458957314491272,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.840624988079071,
+      "step": 432
+    },
+    {
+      "completion_length": 123.296875,
+      "epoch": 0.5773333333333334,
+      "grad_norm": 5.906874029295202,
+      "kl": 0.06689453125,
+      "learning_rate": 7.113333333333334e-07,
+      "loss": 0.0027,
+      "reward": 1.7463542222976685,
+      "reward_std": 0.18887348473072052,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7619791626930237,
+      "step": 433
+    },
+    {
+      "completion_length": 105.09375,
+      "epoch": 0.5786666666666667,
+      "grad_norm": 1.7704652295131011,
+      "kl": 0.099609375,
+      "learning_rate": 7.106666666666666e-07,
+      "loss": 0.004,
+      "reward": 1.8721354007720947,
+      "reward_std": 0.052715495228767395,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8721354603767395,
+      "step": 434
+    },
+    {
+      "completion_length": 112.171875,
+      "epoch": 0.58,
+      "grad_norm": 7.519433282674224,
+      "kl": 0.06787109375,
+      "learning_rate": 7.1e-07,
+      "loss": 0.0027,
+      "reward": 1.8932292461395264,
+      "reward_std": 0.0933760553598404,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8932291865348816,
+      "step": 435
+    },
+    {
+      "completion_length": 115.609375,
+      "epoch": 0.5813333333333334,
+      "grad_norm": 2.6985993580255148,
+      "kl": 0.055908203125,
+      "learning_rate": 7.093333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.838281273841858,
+      "reward_std": 0.035012539476156235,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8382812738418579,
+      "step": 436
+    },
+    {
+      "completion_length": 133.9375,
+      "epoch": 0.5826666666666667,
+      "grad_norm": 3.1898787104063704,
+      "kl": 0.04248046875,
+      "learning_rate": 7.086666666666667e-07,
+      "loss": 0.0017,
+      "reward": 1.7794270515441895,
+      "reward_std": 0.25280001759529114,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8106771111488342,
+      "step": 437
+    },
+    {
+      "completion_length": 118.6875,
+      "epoch": 0.584,
+      "grad_norm": 0.6471469843365322,
+      "kl": 0.04248046875,
+      "learning_rate": 7.079999999999999e-07,
+      "loss": 0.0017,
+      "reward": 1.8802082538604736,
+      "reward_std": 0.06733439117670059,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8958333134651184,
+      "step": 438
+    },
+    {
+      "completion_length": 126.046875,
+      "epoch": 0.5853333333333334,
+      "grad_norm": 77.63453592381838,
+      "kl": 0.05517578125,
+      "learning_rate": 7.073333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.8455729484558105,
+      "reward_std": 0.05421857535839081,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8455728888511658,
+      "step": 439
+    },
+    {
+      "completion_length": 115.796875,
+      "epoch": 0.5866666666666667,
+      "grad_norm": 1.5822923388879424,
+      "kl": 0.040771484375,
+      "learning_rate": 7.066666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.953125,
+      "reward_std": 0.08714609593153,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.953125,
+      "step": 440
+    },
+    {
+      "completion_length": 112.484375,
+      "epoch": 0.588,
+      "grad_norm": 1.648064312887028,
+      "kl": 0.04345703125,
+      "learning_rate": 7.059999999999999e-07,
+      "loss": 0.0017,
+      "reward": 1.8984375,
+      "reward_std": 0.10862711817026138,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8984375,
+      "step": 441
+    },
+    {
+      "completion_length": 121.171875,
+      "epoch": 0.5893333333333334,
+      "grad_norm": 1.1501456775288548,
+      "kl": 0.0390625,
+      "learning_rate": 7.053333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.889062523841858,
+      "reward_std": 0.0677083283662796,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8890625238418579,
+      "step": 442
+    },
+    {
+      "completion_length": 126.546875,
+      "epoch": 0.5906666666666667,
+      "grad_norm": 2.9119187675480083,
+      "kl": 0.039306640625,
+      "learning_rate": 7.046666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.7239583730697632,
+      "reward_std": 0.14186251163482666,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7552083730697632,
+      "step": 443
+    },
+    {
+      "completion_length": 114.90625,
+      "epoch": 0.592,
+      "grad_norm": 11.93091148386865,
+      "kl": 0.029296875,
+      "learning_rate": 7.04e-07,
+      "loss": 0.0012,
+      "reward": 1.8671875,
+      "reward_std": 0.07036440819501877,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8828125,
+      "step": 444
+    },
+    {
+      "completion_length": 125.265625,
+      "epoch": 0.5933333333333334,
+      "grad_norm": 2.9025511349826174,
+      "kl": 0.07080078125,
+      "learning_rate": 7.033333333333333e-07,
+      "loss": 0.0028,
+      "reward": 1.8065104484558105,
+      "reward_std": 0.11598346382379532,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8221354484558105,
+      "step": 445
+    },
+    {
+      "completion_length": 109.796875,
+      "epoch": 0.5946666666666667,
+      "grad_norm": 4.236276118021154,
+      "kl": 0.09228515625,
+      "learning_rate": 7.026666666666667e-07,
+      "loss": 0.0037,
+      "reward": 1.7820312976837158,
+      "reward_std": 0.16205616295337677,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7976562976837158,
+      "step": 446
+    },
+    {
+      "completion_length": 104.0625,
+      "epoch": 0.596,
+      "grad_norm": 24.22867553683186,
+      "kl": 0.0556640625,
+      "learning_rate": 7.019999999999999e-07,
+      "loss": 0.0022,
+      "reward": 1.7708333730697632,
+      "reward_std": 0.08185655623674393,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7708333730697632,
+      "step": 447
+    },
+    {
+      "completion_length": 102.90625,
+      "epoch": 0.5973333333333334,
+      "grad_norm": 1.1824305713748846,
+      "kl": 0.02880859375,
+      "learning_rate": 7.013333333333334e-07,
+      "loss": 0.0012,
+      "reward": 1.881250023841858,
+      "reward_std": 0.07358439266681671,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8812499642372131,
+      "step": 448
+    },
+    {
+      "completion_length": 110.5,
+      "epoch": 0.5986666666666667,
+      "grad_norm": 2.6238053020896754,
+      "kl": 0.054931640625,
+      "learning_rate": 7.006666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.90234375,
+      "reward_std": 0.03281249850988388,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9023438096046448,
+      "step": 449
+    },
+    {
+      "completion_length": 119.8125,
+      "epoch": 0.6,
+      "grad_norm": 2.5622676430051348,
+      "kl": 0.053466796875,
+      "learning_rate": 7e-07,
+      "loss": 0.0021,
+      "reward": 1.7291667461395264,
+      "reward_std": 0.2027510553598404,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7447916269302368,
+      "step": 450
+    },
+    {
+      "completion_length": 108.171875,
+      "epoch": 0.6013333333333334,
+      "grad_norm": 6.277820783386924,
+      "kl": 0.07861328125,
+      "learning_rate": 6.993333333333333e-07,
+      "loss": 0.0031,
+      "reward": 1.8432292938232422,
+      "reward_std": 0.12045939266681671,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8588541746139526,
+      "step": 451
+    },
+    {
+      "completion_length": 106.0625,
+      "epoch": 0.6026666666666667,
+      "grad_norm": 1.4674474823278618,
+      "kl": 0.033447265625,
+      "learning_rate": 6.986666666666667e-07,
+      "loss": 0.0013,
+      "reward": 1.8192708492279053,
+      "reward_std": 0.08248752355575562,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8192707896232605,
+      "step": 452
+    },
+    {
+      "completion_length": 99.71875,
+      "epoch": 0.604,
+      "grad_norm": 1.9806404223831393,
+      "kl": 0.06982421875,
+      "learning_rate": 6.979999999999999e-07,
+      "loss": 0.0028,
+      "reward": 1.685156226158142,
+      "reward_std": 0.10700986534357071,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6851562261581421,
+      "step": 453
+    },
+    {
+      "completion_length": 119.609375,
+      "epoch": 0.6053333333333333,
+      "grad_norm": 1.3389205431826088,
+      "kl": 0.056396484375,
+      "learning_rate": 6.973333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.793229103088379,
+      "reward_std": 0.05312499403953552,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7932292222976685,
+      "step": 454
+    },
+    {
+      "completion_length": 105.1875,
+      "epoch": 0.6066666666666667,
+      "grad_norm": 1.711052259251768,
+      "kl": 0.0673828125,
+      "learning_rate": 6.966666666666666e-07,
+      "loss": 0.0027,
+      "reward": 1.8429687023162842,
+      "reward_std": 0.09906214475631714,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.874218761920929,
+      "step": 455
+    },
+    {
+      "completion_length": 103.5625,
+      "epoch": 0.608,
+      "grad_norm": 2.7002221217989955,
+      "kl": 0.07275390625,
+      "learning_rate": 6.959999999999999e-07,
+      "loss": 0.0029,
+      "reward": 1.6798734664916992,
+      "reward_std": 0.10693712532520294,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.679873526096344,
+      "step": 456
+    },
+    {
+      "completion_length": 115.25,
+      "epoch": 0.6093333333333333,
+      "grad_norm": 1.5630690015396633,
+      "kl": 0.052978515625,
+      "learning_rate": 6.953333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.8418898582458496,
+      "reward_std": 0.03338226303458214,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8418898582458496,
+      "step": 457
+    },
+    {
+      "completion_length": 107.921875,
+      "epoch": 0.6106666666666667,
+      "grad_norm": 5.521358464343591,
+      "kl": 0.044677734375,
+      "learning_rate": 6.946666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.7374999523162842,
+      "reward_std": 0.125,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.753125011920929,
+      "step": 458
+    },
+    {
+      "completion_length": 100.140625,
+      "epoch": 0.612,
+      "grad_norm": 1.6079861053798599,
+      "kl": 0.10791015625,
+      "learning_rate": 6.939999999999999e-07,
+      "loss": 0.0043,
+      "reward": 1.90234375,
+      "reward_std": 0.007812502793967724,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.90234375,
+      "step": 459
+    },
+    {
+      "completion_length": 115.65625,
+      "epoch": 0.6133333333333333,
+      "grad_norm": 1.9125497634634285,
+      "kl": 0.060546875,
+      "learning_rate": 6.933333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.808333396911621,
+      "reward_std": 0.14317253232002258,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8239583969116211,
+      "step": 460
+    },
+    {
+      "completion_length": 103.03125,
+      "epoch": 0.6146666666666667,
+      "grad_norm": 1.966082849073828,
+      "kl": 0.0693359375,
+      "learning_rate": 6.926666666666666e-07,
+      "loss": 0.0028,
+      "reward": 1.7374999523162842,
+      "reward_std": 0.1542895883321762,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7531250715255737,
+      "step": 461
+    },
+    {
+      "completion_length": 113.796875,
+      "epoch": 0.616,
+      "grad_norm": 2.644361779961562,
+      "kl": 0.05029296875,
+      "learning_rate": 6.919999999999999e-07,
+      "loss": 0.002,
+      "reward": 1.9010417461395264,
+      "reward_std": 0.09375,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9010416865348816,
+      "step": 462
+    },
+    {
+      "completion_length": 113.296875,
+      "epoch": 0.6173333333333333,
+      "grad_norm": 1.2125221037970793,
+      "kl": 0.027099609375,
+      "learning_rate": 6.913333333333334e-07,
+      "loss": 0.0011,
+      "reward": 1.90625,
+      "reward_std": 0.13466878235340118,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.90625,
+      "step": 463
+    },
+    {
+      "completion_length": 114.296875,
+      "epoch": 0.6186666666666667,
+      "grad_norm": 0.8789782118597683,
+      "kl": 0.06982421875,
+      "learning_rate": 6.906666666666666e-07,
+      "loss": 0.0028,
+      "reward": 1.8020833730697632,
+      "reward_std": 0.09375,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8177083730697632,
+      "step": 464
+    },
+    {
+      "completion_length": 106.21875,
+      "epoch": 0.62,
+      "grad_norm": 2.5067393668846902,
+      "kl": 0.07177734375,
+      "learning_rate": 6.9e-07,
+      "loss": 0.0029,
+      "reward": 1.7518229484558105,
+      "reward_std": 0.15280002355575562,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7674478888511658,
+      "step": 465
+    },
+    {
+      "completion_length": 96.953125,
+      "epoch": 0.6213333333333333,
+      "grad_norm": 3.283418018949324,
+      "kl": 0.09375,
+      "learning_rate": 6.893333333333333e-07,
+      "loss": 0.0037,
+      "reward": 1.8994419574737549,
+      "reward_std": 0.062314972281455994,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8994420170783997,
+      "step": 466
+    },
+    {
+      "completion_length": 113.5625,
+      "epoch": 0.6226666666666667,
+      "grad_norm": 2.83280095326253,
+      "kl": 0.047607421875,
+      "learning_rate": 6.886666666666667e-07,
+      "loss": 0.0019,
+      "reward": 1.703125,
+      "reward_std": 0.09375,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.71875,
+      "step": 467
+    },
+    {
+      "completion_length": 114.140625,
+      "epoch": 0.624,
+      "grad_norm": 3.003425289761032,
+      "kl": 0.05908203125,
+      "learning_rate": 6.879999999999999e-07,
+      "loss": 0.0024,
+      "reward": 1.7937500476837158,
+      "reward_std": 0.17879271507263184,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8250000476837158,
+      "step": 468
+    },
+    {
+      "completion_length": 115.59375,
+      "epoch": 0.6253333333333333,
+      "grad_norm": 1.495052156674613,
+      "kl": 0.034912109375,
+      "learning_rate": 6.873333333333334e-07,
+      "loss": 0.0014,
+      "reward": 1.7260416746139526,
+      "reward_std": 0.14263354241847992,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7729166746139526,
+      "step": 469
+    },
+    {
+      "completion_length": 113.3125,
+      "epoch": 0.6266666666666667,
+      "grad_norm": 1.7013378438052282,
+      "kl": 0.0830078125,
+      "learning_rate": 6.866666666666666e-07,
+      "loss": 0.0033,
+      "reward": 1.905877947807312,
+      "reward_std": 0.05485360324382782,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.905877947807312,
+      "step": 470
+    },
+    {
+      "completion_length": 114.140625,
+      "epoch": 0.628,
+      "grad_norm": 2.9640504543567228,
+      "kl": 0.057373046875,
+      "learning_rate": 6.86e-07,
+      "loss": 0.0023,
+      "reward": 1.8606771230697632,
+      "reward_std": 0.1063770279288292,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8763020038604736,
+      "step": 471
+    },
+    {
+      "completion_length": 114.359375,
+      "epoch": 0.6293333333333333,
+      "grad_norm": 1.4546078051072522,
+      "kl": 0.040283203125,
+      "learning_rate": 6.853333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.7440104484558105,
+      "reward_std": 0.06406249850988388,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.759635329246521,
+      "step": 472
+    },
+    {
+      "completion_length": 103.265625,
+      "epoch": 0.6306666666666667,
+      "grad_norm": 1.398095231714995,
+      "kl": 0.052978515625,
+      "learning_rate": 6.846666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.91796875,
+      "reward_std": 0.09077189117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.91796875,
+      "step": 473
+    },
+    {
+      "completion_length": 109.953125,
+      "epoch": 0.632,
+      "grad_norm": 0.7008578016301933,
+      "kl": 0.04296875,
+      "learning_rate": 6.84e-07,
+      "loss": 0.0017,
+      "reward": 1.9127604961395264,
+      "reward_std": 0.007812500931322575,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9127604365348816,
+      "step": 474
+    },
+    {
+      "completion_length": 104.3125,
+      "epoch": 0.6333333333333333,
+      "grad_norm": 2.21545700406833,
+      "kl": 0.041259765625,
+      "learning_rate": 6.833333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.8225818872451782,
+      "reward_std": 0.04542006179690361,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8225818276405334,
+      "step": 475
+    },
+    {
+      "completion_length": 107.78125,
+      "epoch": 0.6346666666666667,
+      "grad_norm": 20.216975665963226,
+      "kl": 0.06884765625,
+      "learning_rate": 6.826666666666666e-07,
+      "loss": 0.0028,
+      "reward": 1.792708396911621,
+      "reward_std": 0.1865161806344986,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7927082777023315,
+      "step": 476
+    },
+    {
+      "completion_length": 111.109375,
+      "epoch": 0.636,
+      "grad_norm": 2.2254716047598255,
+      "kl": 0.0615234375,
+      "learning_rate": 6.82e-07,
+      "loss": 0.0025,
+      "reward": 1.8291666507720947,
+      "reward_std": 0.06612156331539154,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8291666507720947,
+      "step": 477
+    },
+    {
+      "completion_length": 104.359375,
+      "epoch": 0.6373333333333333,
+      "grad_norm": 2.980465822831475,
+      "kl": 0.04296875,
+      "learning_rate": 6.813333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.8497395515441895,
+      "reward_std": 0.13444659113883972,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8809895515441895,
+      "step": 478
+    },
+    {
+      "completion_length": 108.125,
+      "epoch": 0.6386666666666667,
+      "grad_norm": 2.3997999449647343,
+      "kl": 0.06787109375,
+      "learning_rate": 6.806666666666666e-07,
+      "loss": 0.0027,
+      "reward": 1.9278273582458496,
+      "reward_std": 0.019931256771087646,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9278273582458496,
+      "step": 479
+    },
+    {
+      "completion_length": 102.296875,
+      "epoch": 0.64,
+      "grad_norm": 1.8841049008881638,
+      "kl": 0.08349609375,
+      "learning_rate": 6.800000000000001e-07,
+      "loss": 0.0033,
+      "reward": 1.8878347873687744,
+      "reward_std": 0.037656694650650024,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8878347873687744,
+      "step": 480
+    },
+    {
+      "completion_length": 110.328125,
+      "epoch": 0.6413333333333333,
+      "grad_norm": 2.0758296903679243,
+      "kl": 0.0625,
+      "learning_rate": 6.793333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.8875000476837158,
+      "reward_std": 0.08958333730697632,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8875000476837158,
+      "step": 481
+    },
+    {
+      "completion_length": 108.0625,
+      "epoch": 0.6426666666666667,
+      "grad_norm": 1.4699837735327201,
+      "kl": 0.0517578125,
+      "learning_rate": 6.786666666666667e-07,
+      "loss": 0.0021,
+      "reward": 1.8026843070983887,
+      "reward_std": 0.16868767142295837,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8183093070983887,
+      "step": 482
+    },
+    {
+      "completion_length": 108.296875,
+      "epoch": 0.644,
+      "grad_norm": 2.073768724329381,
+      "kl": 0.06787109375,
+      "learning_rate": 6.78e-07,
+      "loss": 0.0027,
+      "reward": 1.8854166269302368,
+      "reward_std": 0.09001511335372925,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8854166865348816,
+      "step": 483
+    },
+    {
+      "completion_length": 111.46875,
+      "epoch": 0.6453333333333333,
+      "grad_norm": 2.161251069875967,
+      "kl": 0.043212890625,
+      "learning_rate": 6.773333333333334e-07,
+      "loss": 0.0017,
+      "reward": 1.8229167461395264,
+      "reward_std": 0.12908649444580078,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8229166865348816,
+      "step": 484
+    },
+    {
+      "completion_length": 106.375,
+      "epoch": 0.6466666666666666,
+      "grad_norm": 1.6650478343617205,
+      "kl": 0.0771484375,
+      "learning_rate": 6.766666666666666e-07,
+      "loss": 0.0031,
+      "reward": 1.7395833730697632,
+      "reward_std": 0.09375,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7552083134651184,
+      "step": 485
+    },
+    {
+      "completion_length": 117.15625,
+      "epoch": 0.648,
+      "grad_norm": 2.2162631661789014,
+      "kl": 0.05859375,
+      "learning_rate": 6.76e-07,
+      "loss": 0.0023,
+      "reward": 1.7507812976837158,
+      "reward_std": 0.1164981946349144,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.766406238079071,
+      "step": 486
+    },
+    {
+      "completion_length": 117.828125,
+      "epoch": 0.6493333333333333,
+      "grad_norm": 1.5294970710026992,
+      "kl": 0.041015625,
+      "learning_rate": 6.753333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.9010417461395264,
+      "reward_std": 0.09375,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9010416865348816,
+      "step": 487
+    },
+    {
+      "completion_length": 114.421875,
+      "epoch": 0.6506666666666666,
+      "grad_norm": 4.215186908062091,
+      "kl": 0.087890625,
+      "learning_rate": 6.746666666666666e-07,
+      "loss": 0.0035,
+      "reward": 1.7786458730697632,
+      "reward_std": 0.0885416567325592,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7786458730697632,
+      "step": 488
+    },
+    {
+      "completion_length": 114.40625,
+      "epoch": 0.652,
+      "grad_norm": 2.3673858790928906,
+      "kl": 0.057373046875,
+      "learning_rate": 6.74e-07,
+      "loss": 0.0023,
+      "reward": 1.849218726158142,
+      "reward_std": 0.07884754985570908,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8648437261581421,
+      "step": 489
+    },
+    {
+      "completion_length": 116.796875,
+      "epoch": 0.6533333333333333,
+      "grad_norm": 2.413353919943048,
+      "kl": 0.068359375,
+      "learning_rate": 6.733333333333333e-07,
+      "loss": 0.0027,
+      "reward": 1.7970609664916992,
+      "reward_std": 0.15654602646827698,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.797061026096344,
+      "step": 490
+    },
+    {
+      "completion_length": 109.125,
+      "epoch": 0.6546666666666666,
+      "grad_norm": 0.9372646445695583,
+      "kl": 0.034912109375,
+      "learning_rate": 6.726666666666666e-07,
+      "loss": 0.0014,
+      "reward": 1.9395833015441895,
+      "reward_std": 0.007216875907033682,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.939583420753479,
+      "step": 491
+    },
+    {
+      "completion_length": 113.140625,
+      "epoch": 0.656,
+      "grad_norm": 2.548907017815499,
+      "kl": 0.09228515625,
+      "learning_rate": 6.72e-07,
+      "loss": 0.0037,
+      "reward": 1.7690104246139526,
+      "reward_std": 0.0409083291888237,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7690104842185974,
+      "step": 492
+    },
+    {
+      "completion_length": 124.65625,
+      "epoch": 0.6573333333333333,
+      "grad_norm": 2.0108286325171454,
+      "kl": 0.05419921875,
+      "learning_rate": 6.713333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.80859375,
+      "reward_std": 0.11646566540002823,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.83984375,
+      "step": 493
+    },
+    {
+      "completion_length": 120.046875,
+      "epoch": 0.6586666666666666,
+      "grad_norm": 1.3249092181201774,
+      "kl": 0.046142578125,
+      "learning_rate": 6.706666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.8515625,
+      "reward_std": 0.1037231981754303,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8671875,
+      "step": 494
+    },
+    {
+      "completion_length": 101.359375,
+      "epoch": 0.66,
+      "grad_norm": 1.6214046767101151,
+      "kl": 0.05224609375,
+      "learning_rate": 6.7e-07,
+      "loss": 0.0021,
+      "reward": 1.8807291984558105,
+      "reward_std": 0.11001111567020416,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8807291984558105,
+      "step": 495
+    },
+    {
+      "completion_length": 110.84375,
+      "epoch": 0.6613333333333333,
+      "grad_norm": 15.296042990925761,
+      "kl": 0.055419921875,
+      "learning_rate": 6.693333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.8057291507720947,
+      "reward_std": 0.1387384533882141,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8213541507720947,
+      "step": 496
+    },
+    {
+      "completion_length": 114.984375,
+      "epoch": 0.6626666666666666,
+      "grad_norm": 1.5234859601513446,
+      "kl": 0.04052734375,
+      "learning_rate": 6.686666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.8640997409820557,
+      "reward_std": 0.12849268317222595,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8640997409820557,
+      "step": 497
+    },
+    {
+      "completion_length": 126.65625,
+      "epoch": 0.664,
+      "grad_norm": 13.992725888504415,
+      "kl": 0.035400390625,
+      "learning_rate": 6.68e-07,
+      "loss": 0.0014,
+      "reward": 1.75,
+      "reward_std": 0.09858439117670059,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.78125,
+      "step": 498
+    },
+    {
+      "completion_length": 126.71875,
+      "epoch": 0.6653333333333333,
+      "grad_norm": 1.961666240747246,
+      "kl": 0.053955078125,
+      "learning_rate": 6.673333333333334e-07,
+      "loss": 0.0022,
+      "reward": 1.7999999523162842,
+      "reward_std": 0.1288810521364212,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.800000011920929,
+      "step": 499
+    },
+    {
+      "completion_length": 124.265625,
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.7520966561449056,
+      "kl": 0.054931640625,
+      "learning_rate": 6.666666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.78125,
+      "reward_std": 0.12952522933483124,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.796875,
+      "step": 500
+    },
+    {
+      "completion_length": 125.921875,
+      "epoch": 0.668,
+      "grad_norm": 6.612092455562861,
+      "kl": 0.044677734375,
+      "learning_rate": 6.66e-07,
+      "loss": 0.0018,
+      "reward": 1.7786457538604736,
+      "reward_std": 0.18641570210456848,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8098958134651184,
+      "step": 501
+    },
+    {
+      "completion_length": 114.890625,
+      "epoch": 0.6693333333333333,
+      "grad_norm": 3.041458174020522,
+      "kl": 0.0673828125,
+      "learning_rate": 6.653333333333333e-07,
+      "loss": 0.0027,
+      "reward": 1.7630207538604736,
+      "reward_std": 0.10152731835842133,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7630208134651184,
+      "step": 502
+    },
+    {
+      "completion_length": 125.1875,
+      "epoch": 0.6706666666666666,
+      "grad_norm": 2.57307086398334,
+      "kl": 0.07373046875,
+      "learning_rate": 6.646666666666666e-07,
+      "loss": 0.003,
+      "reward": 1.8187499046325684,
+      "reward_std": 0.13048851490020752,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8500000238418579,
+      "step": 503
+    },
+    {
+      "completion_length": 129.046875,
+      "epoch": 0.672,
+      "grad_norm": 2.4229629726902266,
+      "kl": 0.0556640625,
+      "learning_rate": 6.64e-07,
+      "loss": 0.0022,
+      "reward": 1.8549479246139526,
+      "reward_std": 0.13780389726161957,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8705729246139526,
+      "step": 504
+    },
+    {
+      "completion_length": 115.671875,
+      "epoch": 0.6733333333333333,
+      "grad_norm": 1.8781517192683679,
+      "kl": 0.055419921875,
+      "learning_rate": 6.633333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.865625023841858,
+      "reward_std": 0.054811250418424606,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8656250238418579,
+      "step": 505
+    },
+    {
+      "completion_length": 122.640625,
+      "epoch": 0.6746666666666666,
+      "grad_norm": 4.018861882433775,
+      "kl": 0.036376953125,
+      "learning_rate": 6.626666666666666e-07,
+      "loss": 0.0015,
+      "reward": 1.8033854961395264,
+      "reward_std": 0.14233556389808655,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8033853769302368,
+      "step": 506
+    },
+    {
+      "completion_length": 124.96875,
+      "epoch": 0.676,
+      "grad_norm": 2.7873653322222154,
+      "kl": 0.06591796875,
+      "learning_rate": 6.62e-07,
+      "loss": 0.0026,
+      "reward": 1.7643228769302368,
+      "reward_std": 0.09229008853435516,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7643229365348816,
+      "step": 507
+    },
+    {
+      "completion_length": 120.046875,
+      "epoch": 0.6773333333333333,
+      "grad_norm": 2.0087128178919964,
+      "kl": 0.06298828125,
+      "learning_rate": 6.613333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.9320311546325684,
+      "reward_std": 0.07479298859834671,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9320312142372131,
+      "step": 508
+    },
+    {
+      "completion_length": 119.796875,
+      "epoch": 0.6786666666666666,
+      "grad_norm": 1.815205367354839,
+      "kl": 0.06689453125,
+      "learning_rate": 6.606666666666666e-07,
+      "loss": 0.0027,
+      "reward": 1.8252604007720947,
+      "reward_std": 0.1526620090007782,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8252604007720947,
+      "step": 509
+    },
+    {
+      "completion_length": 125.46875,
+      "epoch": 0.68,
+      "grad_norm": 1.7743059118646152,
+      "kl": 0.044921875,
+      "learning_rate": 6.6e-07,
+      "loss": 0.0018,
+      "reward": 1.6875,
+      "reward_std": 0.11281382292509079,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6875000596046448,
+      "step": 510
+    },
+    {
+      "completion_length": 122.265625,
+      "epoch": 0.6813333333333333,
+      "grad_norm": 4.897996146463126,
+      "kl": 0.07958984375,
+      "learning_rate": 6.593333333333333e-07,
+      "loss": 0.0032,
+      "reward": 1.8152902126312256,
+      "reward_std": 0.09336628019809723,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8152902126312256,
+      "step": 511
+    },
+    {
+      "completion_length": 123.53125,
+      "epoch": 0.6826666666666666,
+      "grad_norm": 1.5325089497485815,
+      "kl": 0.059326171875,
+      "learning_rate": 6.586666666666666e-07,
+      "loss": 0.0024,
+      "reward": 1.8679687976837158,
+      "reward_std": 0.05051518604159355,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8679687976837158,
+      "step": 512
+    },
+    {
+      "completion_length": 119.9375,
+      "epoch": 0.684,
+      "grad_norm": 0.8953670240750193,
+      "kl": 0.03857421875,
+      "learning_rate": 6.58e-07,
+      "loss": 0.0015,
+      "reward": 1.921875,
+      "reward_std": 0.06733439117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.921875,
+      "step": 513
+    },
+    {
+      "completion_length": 126.109375,
+      "epoch": 0.6853333333333333,
+      "grad_norm": 1.3703025518709717,
+      "kl": 0.034423828125,
+      "learning_rate": 6.573333333333333e-07,
+      "loss": 0.0014,
+      "reward": 1.7877604961395264,
+      "reward_std": 0.10639689117670059,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8033854365348816,
+      "step": 514
+    },
+    {
+      "completion_length": 127.09375,
+      "epoch": 0.6866666666666666,
+      "grad_norm": 5.104354354876465,
+      "kl": 0.2431640625,
+      "learning_rate": 6.566666666666666e-07,
+      "loss": 0.0097,
+      "reward": 1.870833396911621,
+      "reward_std": 0.12283650040626526,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8708333373069763,
+      "step": 515
+    },
+    {
+      "completion_length": 133.3125,
+      "epoch": 0.688,
+      "grad_norm": 2.164776479240458,
+      "kl": 0.0517578125,
+      "learning_rate": 6.56e-07,
+      "loss": 0.0021,
+      "reward": 1.7262276411056519,
+      "reward_std": 0.15408842265605927,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7418527007102966,
+      "step": 516
+    },
+    {
+      "completion_length": 130.671875,
+      "epoch": 0.6893333333333334,
+      "grad_norm": 1.5465434603572648,
+      "kl": 0.05615234375,
+      "learning_rate": 6.553333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.9078125953674316,
+      "reward_std": 0.05085911601781845,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9078125357627869,
+      "step": 517
+    },
+    {
+      "completion_length": 129.265625,
+      "epoch": 0.6906666666666667,
+      "grad_norm": 2.1193847828723382,
+      "kl": 0.0556640625,
+      "learning_rate": 6.546666666666665e-07,
+      "loss": 0.0022,
+      "reward": 1.8619792461395264,
+      "reward_std": 0.11845394223928452,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8619791865348816,
+      "step": 518
+    },
+    {
+      "completion_length": 134.09375,
+      "epoch": 0.692,
+      "grad_norm": 1.538395533318019,
+      "kl": 0.046875,
+      "learning_rate": 6.54e-07,
+      "loss": 0.0019,
+      "reward": 1.7008929252624512,
+      "reward_std": 0.1134653389453888,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7477678656578064,
+      "step": 519
+    },
+    {
+      "completion_length": 124.28125,
+      "epoch": 0.6933333333333334,
+      "grad_norm": 1.5648175357037484,
+      "kl": 0.045166015625,
+      "learning_rate": 6.533333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.7630208730697632,
+      "reward_std": 0.2277711033821106,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7786458730697632,
+      "step": 520
+    },
+    {
+      "completion_length": 135.75,
+      "epoch": 0.6946666666666667,
+      "grad_norm": 2.1808609851052045,
+      "kl": 0.0634765625,
+      "learning_rate": 6.526666666666666e-07,
+      "loss": 0.0025,
+      "reward": 1.7736979722976685,
+      "reward_std": 0.11573092639446259,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7893229126930237,
+      "step": 521
+    },
+    {
+      "completion_length": 128.328125,
+      "epoch": 0.696,
+      "grad_norm": 2.0197935995738594,
+      "kl": 0.05224609375,
+      "learning_rate": 6.52e-07,
+      "loss": 0.0021,
+      "reward": 1.7760417461395264,
+      "reward_std": 0.13976788520812988,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7760416865348816,
+      "step": 522
+    },
+    {
+      "completion_length": 125.984375,
+      "epoch": 0.6973333333333334,
+      "grad_norm": 18.975635409587507,
+      "kl": 0.07421875,
+      "learning_rate": 6.513333333333333e-07,
+      "loss": 0.003,
+      "reward": 1.7558965682983398,
+      "reward_std": 0.09847067296504974,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7558965086936951,
+      "step": 523
+    },
+    {
+      "completion_length": 129.546875,
+      "epoch": 0.6986666666666667,
+      "grad_norm": 2.1535415020336224,
+      "kl": 0.0478515625,
+      "learning_rate": 6.506666666666666e-07,
+      "loss": 0.0019,
+      "reward": 1.7861979007720947,
+      "reward_std": 0.2003350704908371,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8330729007720947,
+      "step": 524
+    },
+    {
+      "completion_length": 144.234375,
+      "epoch": 0.7,
+      "grad_norm": 3.1236612108795176,
+      "kl": 0.0556640625,
+      "learning_rate": 6.5e-07,
+      "loss": 0.0022,
+      "reward": 1.7369791269302368,
+      "reward_std": 0.17545056343078613,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7838541865348816,
+      "step": 525
+    },
+    {
+      "completion_length": 147.09375,
+      "epoch": 0.7013333333333334,
+      "grad_norm": 1.4740832854223405,
+      "kl": 0.060302734375,
+      "learning_rate": 6.493333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.8452754020690918,
+      "reward_std": 0.12467299401760101,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8609002828598022,
+      "step": 526
+    },
+    {
+      "completion_length": 143.09375,
+      "epoch": 0.7026666666666667,
+      "grad_norm": 1.4848476401173474,
+      "kl": 0.049072265625,
+      "learning_rate": 6.486666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.745628833770752,
+      "reward_std": 0.1517936885356903,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7768787741661072,
+      "step": 527
+    },
+    {
+      "completion_length": 128.0,
+      "epoch": 0.704,
+      "grad_norm": 1.4895952070971061,
+      "kl": 0.0439453125,
+      "learning_rate": 6.48e-07,
+      "loss": 0.0018,
+      "reward": 1.906640648841858,
+      "reward_std": 0.12064988911151886,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9222656488418579,
+      "step": 528
+    },
+    {
+      "completion_length": 128.203125,
+      "epoch": 0.7053333333333334,
+      "grad_norm": 1.7468344916225846,
+      "kl": 0.06103515625,
+      "learning_rate": 6.473333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.798437476158142,
+      "reward_std": 0.029294900596141815,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7984374761581421,
+      "step": 529
+    },
+    {
+      "completion_length": 128.328125,
+      "epoch": 0.7066666666666667,
+      "grad_norm": 1.4022882700774622,
+      "kl": 0.041259765625,
+      "learning_rate": 6.466666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.783593773841858,
+      "reward_std": 0.10581940412521362,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7835937738418579,
+      "step": 530
+    },
+    {
+      "completion_length": 131.015625,
+      "epoch": 0.708,
+      "grad_norm": 2.0994265862320716,
+      "kl": 0.04736328125,
+      "learning_rate": 6.46e-07,
+      "loss": 0.0019,
+      "reward": 1.8684896230697632,
+      "reward_std": 0.0651041641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8684896230697632,
+      "step": 531
+    },
+    {
+      "completion_length": 137.171875,
+      "epoch": 0.7093333333333334,
+      "grad_norm": 2.1843032729932923,
+      "kl": 0.056640625,
+      "learning_rate": 6.453333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.7926338911056519,
+      "reward_std": 0.10192298144102097,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8082589507102966,
+      "step": 532
+    },
+    {
+      "completion_length": 132.03125,
+      "epoch": 0.7106666666666667,
+      "grad_norm": 1.8316375196797716,
+      "kl": 0.056640625,
+      "learning_rate": 6.446666666666666e-07,
+      "loss": 0.0023,
+      "reward": 1.882552146911621,
+      "reward_std": 0.034008897840976715,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8825521469116211,
+      "step": 533
+    },
+    {
+      "completion_length": 122.984375,
+      "epoch": 0.712,
+      "grad_norm": 1.5028783061465896,
+      "kl": 0.0390625,
+      "learning_rate": 6.44e-07,
+      "loss": 0.0016,
+      "reward": 1.8875000476837158,
+      "reward_std": 0.13883544504642487,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8874999284744263,
+      "step": 534
+    },
+    {
+      "completion_length": 140.28125,
+      "epoch": 0.7133333333333334,
+      "grad_norm": 4.229730465594616,
+      "kl": 0.044677734375,
+      "learning_rate": 6.433333333333332e-07,
+      "loss": 0.0018,
+      "reward": 1.8606771230697632,
+      "reward_std": 0.10387445986270905,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8763020634651184,
+      "step": 535
+    },
+    {
+      "completion_length": 125.71875,
+      "epoch": 0.7146666666666667,
+      "grad_norm": 1.8714463878686858,
+      "kl": 0.032470703125,
+      "learning_rate": 6.426666666666667e-07,
+      "loss": 0.0013,
+      "reward": 1.9479167461395264,
+      "reward_std": 0.0572916641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9479166865348816,
+      "step": 536
+    },
+    {
+      "completion_length": 126.71875,
+      "epoch": 0.716,
+      "grad_norm": 2.003324125676875,
+      "kl": 0.043701171875,
+      "learning_rate": 6.42e-07,
+      "loss": 0.0017,
+      "reward": 1.8567707538604736,
+      "reward_std": 0.0761338621377945,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8567708134651184,
+      "step": 537
+    },
+    {
+      "completion_length": 117.984375,
+      "epoch": 0.7173333333333334,
+      "grad_norm": 2.3046703922832017,
+      "kl": 0.052001953125,
+      "learning_rate": 6.413333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.8182291984558105,
+      "reward_std": 0.019775627180933952,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8182291984558105,
+      "step": 538
+    },
+    {
+      "completion_length": 126.1875,
+      "epoch": 0.7186666666666667,
+      "grad_norm": 1.8377188638318276,
+      "kl": 0.06103515625,
+      "learning_rate": 6.406666666666667e-07,
+      "loss": 0.0024,
+      "reward": 1.8653645515441895,
+      "reward_std": 0.07554396986961365,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8653646111488342,
+      "step": 539
+    },
+    {
+      "completion_length": 128.21875,
+      "epoch": 0.72,
+      "grad_norm": 1.4001178593703596,
+      "kl": 0.051025390625,
+      "learning_rate": 6.4e-07,
+      "loss": 0.002,
+      "reward": 1.773958444595337,
+      "reward_std": 0.10483439266681671,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7739583849906921,
+      "step": 540
+    },
+    {
+      "completion_length": 141.0625,
+      "epoch": 0.7213333333333334,
+      "grad_norm": 1.4077923195670918,
+      "kl": 0.04052734375,
+      "learning_rate": 6.393333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.7838542461395264,
+      "reward_std": 0.1655070185661316,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.8463541865348816,
+      "step": 541
+    },
+    {
+      "completion_length": 131.078125,
+      "epoch": 0.7226666666666667,
+      "grad_norm": 1.1934185127282155,
+      "kl": 0.040283203125,
+      "learning_rate": 6.386666666666667e-07,
+      "loss": 0.0016,
+      "reward": 1.9265625476837158,
+      "reward_std": 0.06515312939882278,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9265625476837158,
+      "step": 542
+    },
+    {
+      "completion_length": 128.703125,
+      "epoch": 0.724,
+      "grad_norm": 1.5872537648175267,
+      "kl": 0.06494140625,
+      "learning_rate": 6.38e-07,
+      "loss": 0.0026,
+      "reward": 1.8101191520690918,
+      "reward_std": 0.03830573335289955,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8101190328598022,
+      "step": 543
+    },
+    {
+      "completion_length": 135.71875,
+      "epoch": 0.7253333333333334,
+      "grad_norm": 2.393073061180275,
+      "kl": 0.0625,
+      "learning_rate": 6.373333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.7494791746139526,
+      "reward_std": 0.1436351239681244,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7651041746139526,
+      "step": 544
+    },
+    {
+      "completion_length": 121.90625,
+      "epoch": 0.7266666666666667,
+      "grad_norm": 3.0068551041864446,
+      "kl": 0.08642578125,
+      "learning_rate": 6.366666666666667e-07,
+      "loss": 0.0035,
+      "reward": 1.726711392402649,
+      "reward_std": 0.12125444412231445,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7267113327980042,
+      "step": 545
+    },
+    {
+      "completion_length": 121.609375,
+      "epoch": 0.728,
+      "grad_norm": 1.8065947726206384,
+      "kl": 0.043701171875,
+      "learning_rate": 6.36e-07,
+      "loss": 0.0017,
+      "reward": 1.7825521230697632,
+      "reward_std": 0.15483690798282623,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7981770634651184,
+      "step": 546
+    },
+    {
+      "completion_length": 123.703125,
+      "epoch": 0.7293333333333333,
+      "grad_norm": 1.4205073302153828,
+      "kl": 0.040283203125,
+      "learning_rate": 6.353333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.792708396911621,
+      "reward_std": 0.09771046042442322,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8239583373069763,
+      "step": 547
+    },
+    {
+      "completion_length": 122.671875,
+      "epoch": 0.7306666666666667,
+      "grad_norm": 1.5319831274278093,
+      "kl": 0.06396484375,
+      "learning_rate": 6.346666666666666e-07,
+      "loss": 0.0026,
+      "reward": 1.777083396911621,
+      "reward_std": 0.11501513421535492,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7770833969116211,
+      "step": 548
+    },
+    {
+      "completion_length": 115.796875,
+      "epoch": 0.732,
+      "grad_norm": 1.0585845379912298,
+      "kl": 0.0478515625,
+      "learning_rate": 6.34e-07,
+      "loss": 0.0019,
+      "reward": 1.796875,
+      "reward_std": 0.07452812790870667,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.796875,
+      "step": 549
+    },
+    {
+      "completion_length": 115.703125,
+      "epoch": 0.7333333333333333,
+      "grad_norm": 1.3823032034588931,
+      "kl": 0.042236328125,
+      "learning_rate": 6.333333333333332e-07,
+      "loss": 0.0017,
+      "reward": 1.7947916984558105,
+      "reward_std": 0.09753209352493286,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8104166388511658,
+      "step": 550
+    },
+    {
+      "completion_length": 124.046875,
+      "epoch": 0.7346666666666667,
+      "grad_norm": 0.7929432346971809,
+      "kl": 0.041748046875,
+      "learning_rate": 6.326666666666667e-07,
+      "loss": 0.0017,
+      "reward": 1.765625,
+      "reward_std": 0.12233919650316238,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.796875,
+      "step": 551
+    },
+    {
+      "completion_length": 118.3125,
+      "epoch": 0.736,
+      "grad_norm": 0.9260416516378777,
+      "kl": 0.040771484375,
+      "learning_rate": 6.319999999999999e-07,
+      "loss": 0.0016,
+      "reward": 1.8489583730697632,
+      "reward_std": 0.07757841050624847,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8645833134651184,
+      "step": 552
+    },
+    {
+      "completion_length": 112.8125,
+      "epoch": 0.7373333333333333,
+      "grad_norm": 2.4471352835392794,
+      "kl": 0.048828125,
+      "learning_rate": 6.313333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.8562500476837158,
+      "reward_std": 0.08934487402439117,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8562500476837158,
+      "step": 553
+    },
+    {
+      "completion_length": 130.25,
+      "epoch": 0.7386666666666667,
+      "grad_norm": 1.4891038215036443,
+      "kl": 0.038330078125,
+      "learning_rate": 6.306666666666666e-07,
+      "loss": 0.0015,
+      "reward": 1.9215773344039917,
+      "reward_std": 0.08715443313121796,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9372024536132812,
+      "step": 554
+    },
+    {
+      "completion_length": 134.28125,
+      "epoch": 0.74,
+      "grad_norm": 1.0200702544306002,
+      "kl": 0.050048828125,
+      "learning_rate": 6.3e-07,
+      "loss": 0.002,
+      "reward": 1.8020833730697632,
+      "reward_std": 0.0725773274898529,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8020833134651184,
+      "step": 555
+    },
+    {
+      "completion_length": 144.46875,
+      "epoch": 0.7413333333333333,
+      "grad_norm": 3.220705560602401,
+      "kl": 0.049072265625,
+      "learning_rate": 6.293333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.613541603088379,
+      "reward_std": 0.2514597773551941,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.6916666626930237,
+      "step": 556
+    },
+    {
+      "completion_length": 130.859375,
+      "epoch": 0.7426666666666667,
+      "grad_norm": 2.234506618763232,
+      "kl": 0.05224609375,
+      "learning_rate": 6.286666666666667e-07,
+      "loss": 0.0021,
+      "reward": 1.6729166507720947,
+      "reward_std": 0.14995183050632477,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7354167103767395,
+      "step": 557
+    },
+    {
+      "completion_length": 141.484375,
+      "epoch": 0.744,
+      "grad_norm": 1.5188721908893847,
+      "kl": 0.03515625,
+      "learning_rate": 6.28e-07,
+      "loss": 0.0014,
+      "reward": 1.650781273841858,
+      "reward_std": 0.15502884984016418,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7132812738418579,
+      "step": 558
+    },
+    {
+      "completion_length": 124.28125,
+      "epoch": 0.7453333333333333,
+      "grad_norm": 1.9779493453944805,
+      "kl": 0.032958984375,
+      "learning_rate": 6.273333333333333e-07,
+      "loss": 0.0013,
+      "reward": 1.8645833730697632,
+      "reward_std": 0.16108438372612,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8802083134651184,
+      "step": 559
+    },
+    {
+      "completion_length": 136.96875,
+      "epoch": 0.7466666666666667,
+      "grad_norm": 1.114482668908182,
+      "kl": 0.04736328125,
+      "learning_rate": 6.266666666666667e-07,
+      "loss": 0.0019,
+      "reward": 1.875,
+      "reward_std": 0.052515123039484024,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.875,
+      "step": 560
+    },
+    {
+      "completion_length": 126.28125,
+      "epoch": 0.748,
+      "grad_norm": 1.681632309261072,
+      "kl": 0.059814453125,
+      "learning_rate": 6.26e-07,
+      "loss": 0.0024,
+      "reward": 1.8002605438232422,
+      "reward_std": 0.0921127051115036,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8002604246139526,
+      "step": 561
+    },
+    {
+      "completion_length": 119.15625,
+      "epoch": 0.7493333333333333,
+      "grad_norm": 2.469818022468582,
+      "kl": 0.06298828125,
+      "learning_rate": 6.253333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.7330729961395264,
+      "reward_std": 0.17013441026210785,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7643228769302368,
+      "step": 562
+    },
+    {
+      "completion_length": 133.96875,
+      "epoch": 0.7506666666666667,
+      "grad_norm": 1.9475543309779804,
+      "kl": 0.037353515625,
+      "learning_rate": 6.246666666666667e-07,
+      "loss": 0.0015,
+      "reward": 1.855208396911621,
+      "reward_std": 0.09346439689397812,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8708333373069763,
+      "step": 563
+    },
+    {
+      "completion_length": 110.25,
+      "epoch": 0.752,
+      "grad_norm": 1.2965628220078784,
+      "kl": 0.047119140625,
+      "learning_rate": 6.24e-07,
+      "loss": 0.0019,
+      "reward": 1.921875,
+      "reward_std": 0.06589487940073013,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.921875,
+      "step": 564
+    },
+    {
+      "completion_length": 144.46875,
+      "epoch": 0.7533333333333333,
+      "grad_norm": 2.066822211750995,
+      "kl": 0.07470703125,
+      "learning_rate": 6.233333333333332e-07,
+      "loss": 0.003,
+      "reward": 1.7924479246139526,
+      "reward_std": 0.10324320942163467,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8080729246139526,
+      "step": 565
+    },
+    {
+      "completion_length": 136.5,
+      "epoch": 0.7546666666666667,
+      "grad_norm": 2.097223797529753,
+      "kl": 0.058349609375,
+      "learning_rate": 6.226666666666667e-07,
+      "loss": 0.0023,
+      "reward": 1.7744791507720947,
+      "reward_std": 0.09668193012475967,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8057291507720947,
+      "step": 566
+    },
+    {
+      "completion_length": 131.25,
+      "epoch": 0.756,
+      "grad_norm": 10.406413759307506,
+      "kl": 0.064453125,
+      "learning_rate": 6.219999999999999e-07,
+      "loss": 0.0026,
+      "reward": 1.8916666507720947,
+      "reward_std": 0.11477918922901154,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8916666507720947,
+      "step": 567
+    },
+    {
+      "completion_length": 124.484375,
+      "epoch": 0.7573333333333333,
+      "grad_norm": 0.6485990184306736,
+      "kl": 0.0294189453125,
+      "learning_rate": 6.213333333333333e-07,
+      "loss": 0.0012,
+      "reward": 1.96484375,
+      "reward_std": 0.04389689117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.96484375,
+      "step": 568
+    },
+    {
+      "completion_length": 117.359375,
+      "epoch": 0.7586666666666667,
+      "grad_norm": 1.36126122235281,
+      "kl": 0.06787109375,
+      "learning_rate": 6.206666666666666e-07,
+      "loss": 0.0027,
+      "reward": 1.8916666507720947,
+      "reward_std": 0.06875000149011612,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8916667103767395,
+      "step": 569
+    },
+    {
+      "completion_length": 127.296875,
+      "epoch": 0.76,
+      "grad_norm": 1.873074982015224,
+      "kl": 0.060302734375,
+      "learning_rate": 6.2e-07,
+      "loss": 0.0024,
+      "reward": 1.67578125,
+      "reward_std": 0.12282761931419373,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.72265625,
+      "step": 570
+    },
+    {
+      "completion_length": 128.28125,
+      "epoch": 0.7613333333333333,
+      "grad_norm": 2.194902758346219,
+      "kl": 0.064453125,
+      "learning_rate": 6.193333333333332e-07,
+      "loss": 0.0026,
+      "reward": 1.8199219703674316,
+      "reward_std": 0.14801767468452454,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8355469107627869,
+      "step": 571
+    },
+    {
+      "completion_length": 130.515625,
+      "epoch": 0.7626666666666667,
+      "grad_norm": 1.6636687406761956,
+      "kl": 0.0537109375,
+      "learning_rate": 6.186666666666667e-07,
+      "loss": 0.0021,
+      "reward": 1.8466145992279053,
+      "reward_std": 0.05757656320929527,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8466145992279053,
+      "step": 572
+    },
+    {
+      "completion_length": 129.0,
+      "epoch": 0.764,
+      "grad_norm": 3.37936808175803,
+      "kl": 0.053466796875,
+      "learning_rate": 6.18e-07,
+      "loss": 0.0021,
+      "reward": 1.7955729961395264,
+      "reward_std": 0.12769825756549835,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7955728769302368,
+      "step": 573
+    },
+    {
+      "completion_length": 128.53125,
+      "epoch": 0.7653333333333333,
+      "grad_norm": 1.6336587903762736,
+      "kl": 0.06982421875,
+      "learning_rate": 6.173333333333333e-07,
+      "loss": 0.0028,
+      "reward": 1.8117188215255737,
+      "reward_std": 0.10518528521060944,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.811718761920929,
+      "step": 574
+    },
+    {
+      "completion_length": 122.125,
+      "epoch": 0.7666666666666667,
+      "grad_norm": 1.6297927083459822,
+      "kl": 0.0673828125,
+      "learning_rate": 6.166666666666667e-07,
+      "loss": 0.0027,
+      "reward": 1.726822853088379,
+      "reward_std": 0.1264054775238037,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7268229126930237,
+      "step": 575
+    },
+    {
+      "completion_length": 121.21875,
+      "epoch": 0.768,
+      "grad_norm": 3.1752613711230264,
+      "kl": 0.0703125,
+      "learning_rate": 6.16e-07,
+      "loss": 0.0028,
+      "reward": 1.8205729722976685,
+      "reward_std": 0.06878800690174103,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8205729126930237,
+      "step": 576
+    },
+    {
+      "completion_length": 118.8125,
+      "epoch": 0.7693333333333333,
+      "grad_norm": 0.9989862231915262,
+      "kl": 0.059326171875,
+      "learning_rate": 6.153333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.7833333015441895,
+      "reward_std": 0.017633551731705666,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7833333611488342,
+      "step": 577
+    },
+    {
+      "completion_length": 133.515625,
+      "epoch": 0.7706666666666667,
+      "grad_norm": 2.1249983110530586,
+      "kl": 0.041748046875,
+      "learning_rate": 6.146666666666667e-07,
+      "loss": 0.0017,
+      "reward": 1.7744791507720947,
+      "reward_std": 0.23923011124134064,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8213541507720947,
+      "step": 578
+    },
+    {
+      "completion_length": 129.3125,
+      "epoch": 0.772,
+      "grad_norm": 1.4830827615555664,
+      "kl": 0.0634765625,
+      "learning_rate": 6.14e-07,
+      "loss": 0.0025,
+      "reward": 1.732812523841858,
+      "reward_std": 0.14187410473823547,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7484375238418579,
+      "step": 579
+    },
+    {
+      "completion_length": 142.734375,
+      "epoch": 0.7733333333333333,
+      "grad_norm": 1.7400755776906882,
+      "kl": 0.062255859375,
+      "learning_rate": 6.133333333333332e-07,
+      "loss": 0.0025,
+      "reward": 1.7437500953674316,
+      "reward_std": 0.1279469132423401,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7593750357627869,
+      "step": 580
+    },
+    {
+      "completion_length": 124.140625,
+      "epoch": 0.7746666666666666,
+      "grad_norm": 1.5029148278158306,
+      "kl": 0.055908203125,
+      "learning_rate": 6.126666666666667e-07,
+      "loss": 0.0022,
+      "reward": 1.8619792461395264,
+      "reward_std": 0.054500531405210495,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8619791865348816,
+      "step": 581
+    },
+    {
+      "completion_length": 133.796875,
+      "epoch": 0.776,
+      "grad_norm": 5.715852722273617,
+      "kl": 0.23828125,
+      "learning_rate": 6.119999999999999e-07,
+      "loss": 0.0095,
+      "reward": 1.7072917222976685,
+      "reward_std": 0.17553308606147766,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7072916626930237,
+      "step": 582
+    },
+    {
+      "completion_length": 115.109375,
+      "epoch": 0.7773333333333333,
+      "grad_norm": 0.48390883014907043,
+      "kl": 0.031494140625,
+      "learning_rate": 6.113333333333333e-07,
+      "loss": 0.0013,
+      "reward": 1.953125,
+      "reward_std": 0.03125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.953125,
+      "step": 583
+    },
+    {
+      "completion_length": 138.09375,
+      "epoch": 0.7786666666666666,
+      "grad_norm": 1.4359629251073016,
+      "kl": 0.050048828125,
+      "learning_rate": 6.106666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.7981771230697632,
+      "reward_std": 0.04991095885634422,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8294270634651184,
+      "step": 584
+    },
+    {
+      "completion_length": 146.9375,
+      "epoch": 0.78,
+      "grad_norm": 3.5641252783332886,
+      "kl": 0.051025390625,
+      "learning_rate": 6.1e-07,
+      "loss": 0.002,
+      "reward": 1.7262649536132812,
+      "reward_std": 0.21865887939929962,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7731399536132812,
+      "step": 585
+    },
+    {
+      "completion_length": 126.078125,
+      "epoch": 0.7813333333333333,
+      "grad_norm": 0.9878676893415872,
+      "kl": 0.049072265625,
+      "learning_rate": 6.093333333333332e-07,
+      "loss": 0.002,
+      "reward": 1.9187500476837158,
+      "reward_std": 0.03750000149011612,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9187500476837158,
+      "step": 586
+    },
+    {
+      "completion_length": 110.671875,
+      "epoch": 0.7826666666666666,
+      "grad_norm": 1.244986766907493,
+      "kl": 0.032470703125,
+      "learning_rate": 6.086666666666667e-07,
+      "loss": 0.0013,
+      "reward": 1.954545497894287,
+      "reward_std": 0.049242421984672546,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9545454382896423,
+      "step": 587
+    },
+    {
+      "completion_length": 152.234375,
+      "epoch": 0.784,
+      "grad_norm": 1.1804042331851294,
+      "kl": 0.04345703125,
+      "learning_rate": 6.079999999999999e-07,
+      "loss": 0.0017,
+      "reward": 1.553125023841858,
+      "reward_std": 0.20810437202453613,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.6312500238418579,
+      "step": 588
+    },
+    {
+      "completion_length": 136.078125,
+      "epoch": 0.7853333333333333,
+      "grad_norm": 1.3613015404379813,
+      "kl": 0.048095703125,
+      "learning_rate": 6.073333333333333e-07,
+      "loss": 0.0019,
+      "reward": 1.8813244104385376,
+      "reward_std": 0.08281568437814713,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8813244104385376,
+      "step": 589
+    },
+    {
+      "completion_length": 139.65625,
+      "epoch": 0.7866666666666666,
+      "grad_norm": 3.249889548336712,
+      "kl": 0.04931640625,
+      "learning_rate": 6.066666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.7779970169067383,
+      "reward_std": 0.11890867352485657,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8092470169067383,
+      "step": 590
+    },
+    {
+      "completion_length": 128.703125,
+      "epoch": 0.788,
+      "grad_norm": 1.3261780436902222,
+      "kl": 0.0615234375,
+      "learning_rate": 6.06e-07,
+      "loss": 0.0025,
+      "reward": 1.7825521230697632,
+      "reward_std": 0.11123128235340118,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7825521230697632,
+      "step": 591
+    },
+    {
+      "completion_length": 125.609375,
+      "epoch": 0.7893333333333333,
+      "grad_norm": 2.1075070103847646,
+      "kl": 0.0517578125,
+      "learning_rate": 6.053333333333332e-07,
+      "loss": 0.0021,
+      "reward": 1.7726562023162842,
+      "reward_std": 0.14534729719161987,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.772656261920929,
+      "step": 592
+    },
+    {
+      "completion_length": 143.46875,
+      "epoch": 0.7906666666666666,
+      "grad_norm": 1.5128410174974043,
+      "kl": 0.045654296875,
+      "learning_rate": 6.046666666666667e-07,
+      "loss": 0.0018,
+      "reward": 1.6981815099716187,
+      "reward_std": 0.13972139358520508,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7138065099716187,
+      "step": 593
+    },
+    {
+      "completion_length": 138.78125,
+      "epoch": 0.792,
+      "grad_norm": 4.7636423234491305,
+      "kl": 0.040771484375,
+      "learning_rate": 6.04e-07,
+      "loss": 0.0016,
+      "reward": 1.83984375,
+      "reward_std": 0.11535260826349258,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.87109375,
+      "step": 594
+    },
+    {
+      "completion_length": 146.171875,
+      "epoch": 0.7933333333333333,
+      "grad_norm": 1.9234635763749584,
+      "kl": 0.072265625,
+      "learning_rate": 6.033333333333333e-07,
+      "loss": 0.0029,
+      "reward": 1.8230655193328857,
+      "reward_std": 0.0927305817604065,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.838690459728241,
+      "step": 595
+    },
+    {
+      "completion_length": 133.296875,
+      "epoch": 0.7946666666666666,
+      "grad_norm": 2.9702534008047143,
+      "kl": 0.046142578125,
+      "learning_rate": 6.026666666666667e-07,
+      "loss": 0.0018,
+      "reward": 1.7421875,
+      "reward_std": 0.14545938372612,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7421875,
+      "step": 596
+    },
+    {
+      "completion_length": 135.125,
+      "epoch": 0.796,
+      "grad_norm": 1.5330676150517148,
+      "kl": 0.04638671875,
+      "learning_rate": 6.019999999999999e-07,
+      "loss": 0.0019,
+      "reward": 1.8537201881408691,
+      "reward_std": 0.09466174989938736,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8693452477455139,
+      "step": 597
+    },
+    {
+      "completion_length": 155.75,
+      "epoch": 0.7973333333333333,
+      "grad_norm": 2.0462155623180016,
+      "kl": 0.0517578125,
+      "learning_rate": 6.013333333333334e-07,
+      "loss": 0.0021,
+      "reward": 1.7909598350524902,
+      "reward_std": 0.11289841681718826,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8065848350524902,
+      "step": 598
+    },
+    {
+      "completion_length": 146.203125,
+      "epoch": 0.7986666666666666,
+      "grad_norm": 1.7994674071498862,
+      "kl": 0.05224609375,
+      "learning_rate": 6.006666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.8178385496139526,
+      "reward_std": 0.15889057517051697,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8334635496139526,
+      "step": 599
+    },
+    {
+      "completion_length": 148.109375,
+      "epoch": 0.8,
+      "grad_norm": 3.4980586498474655,
+      "kl": 0.09130859375,
+      "learning_rate": 6e-07,
+      "loss": 0.0036,
+      "reward": 1.729873538017273,
+      "reward_std": 0.11851286143064499,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.729873538017273,
+      "step": 600
+    },
+    {
+      "completion_length": 147.0625,
+      "epoch": 0.8013333333333333,
+      "grad_norm": 2.980991656129261,
+      "kl": 0.0703125,
+      "learning_rate": 5.993333333333333e-07,
+      "loss": 0.0028,
+      "reward": 1.719024658203125,
+      "reward_std": 0.2526256740093231,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7502745985984802,
+      "step": 601
+    },
+    {
+      "completion_length": 142.109375,
+      "epoch": 0.8026666666666666,
+      "grad_norm": 1.4912241782704825,
+      "kl": 0.056640625,
+      "learning_rate": 5.986666666666667e-07,
+      "loss": 0.0023,
+      "reward": 1.719010353088379,
+      "reward_std": 0.1544685661792755,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7502604126930237,
+      "step": 602
+    },
+    {
+      "completion_length": 134.375,
+      "epoch": 0.804,
+      "grad_norm": 1.3147210814038246,
+      "kl": 0.04638671875,
+      "learning_rate": 5.979999999999999e-07,
+      "loss": 0.0019,
+      "reward": 1.806249976158142,
+      "reward_std": 0.07839563488960266,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8062500357627869,
+      "step": 603
+    },
+    {
+      "completion_length": 128.46875,
+      "epoch": 0.8053333333333333,
+      "grad_norm": 1.587909174822523,
+      "kl": 0.044189453125,
+      "learning_rate": 5.973333333333334e-07,
+      "loss": 0.0018,
+      "reward": 1.8503063917160034,
+      "reward_std": 0.1418999433517456,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8659313917160034,
+      "step": 604
+    },
+    {
+      "completion_length": 149.875,
+      "epoch": 0.8066666666666666,
+      "grad_norm": 2.294819565227276,
+      "kl": 0.051025390625,
+      "learning_rate": 5.966666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.8411457538604736,
+      "reward_std": 0.07472182810306549,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8567708730697632,
+      "step": 605
+    },
+    {
+      "completion_length": 144.734375,
+      "epoch": 0.808,
+      "grad_norm": 1.304740215616161,
+      "kl": 0.053466796875,
+      "learning_rate": 5.96e-07,
+      "loss": 0.0021,
+      "reward": 1.734114646911621,
+      "reward_std": 0.09454174339771271,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7497395873069763,
+      "step": 606
+    },
+    {
+      "completion_length": 141.546875,
+      "epoch": 0.8093333333333333,
+      "grad_norm": 2.0574914906743866,
+      "kl": 0.0341796875,
+      "learning_rate": 5.953333333333333e-07,
+      "loss": 0.0014,
+      "reward": 1.879166603088379,
+      "reward_std": 0.0804210975766182,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8791666626930237,
+      "step": 607
+    },
+    {
+      "completion_length": 130.40625,
+      "epoch": 0.8106666666666666,
+      "grad_norm": 1.70608068508952,
+      "kl": 0.04248046875,
+      "learning_rate": 5.946666666666667e-07,
+      "loss": 0.0017,
+      "reward": 1.8171875476837158,
+      "reward_std": 0.056025635451078415,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8171875476837158,
+      "step": 608
+    },
+    {
+      "completion_length": 145.609375,
+      "epoch": 0.812,
+      "grad_norm": 1.7159897550876027,
+      "kl": 0.0478515625,
+      "learning_rate": 5.939999999999999e-07,
+      "loss": 0.0019,
+      "reward": 1.7160255908966064,
+      "reward_std": 0.13735981285572052,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7316506505012512,
+      "step": 609
+    },
+    {
+      "completion_length": 135.453125,
+      "epoch": 0.8133333333333334,
+      "grad_norm": 2.524035472026379,
+      "kl": 0.043212890625,
+      "learning_rate": 5.933333333333334e-07,
+      "loss": 0.0017,
+      "reward": 1.88671875,
+      "reward_std": 0.17373126745224,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.91796875,
+      "step": 610
+    },
+    {
+      "completion_length": 122.25,
+      "epoch": 0.8146666666666667,
+      "grad_norm": 2.4447481613054842,
+      "kl": 0.0458984375,
+      "learning_rate": 5.926666666666667e-07,
+      "loss": 0.0018,
+      "reward": 1.924218773841858,
+      "reward_std": 0.07900021970272064,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9242187738418579,
+      "step": 611
+    },
+    {
+      "completion_length": 136.09375,
+      "epoch": 0.816,
+      "grad_norm": 2.3773638076639787,
+      "kl": 0.04443359375,
+      "learning_rate": 5.919999999999999e-07,
+      "loss": 0.0018,
+      "reward": 1.7526042461395264,
+      "reward_std": 0.15029378235340118,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7526041269302368,
+      "step": 612
+    },
+    {
+      "completion_length": 119.234375,
+      "epoch": 0.8173333333333334,
+      "grad_norm": 3.941072143219075,
+      "kl": 0.046630859375,
+      "learning_rate": 5.913333333333334e-07,
+      "loss": 0.0019,
+      "reward": 1.844010353088379,
+      "reward_std": 0.06144700199365616,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8440104722976685,
+      "step": 613
+    },
+    {
+      "completion_length": 138.296875,
+      "epoch": 0.8186666666666667,
+      "grad_norm": 3.07045886422427,
+      "kl": 0.04931640625,
+      "learning_rate": 5.906666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.7258954048156738,
+      "reward_std": 0.16502586007118225,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7571454048156738,
+      "step": 614
+    },
+    {
+      "completion_length": 143.984375,
+      "epoch": 0.82,
+      "grad_norm": 1.981144238825312,
+      "kl": 0.044921875,
+      "learning_rate": 5.9e-07,
+      "loss": 0.0018,
+      "reward": 1.6843750476837158,
+      "reward_std": 0.1547490954399109,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7000000476837158,
+      "step": 615
+    },
+    {
+      "completion_length": 128.921875,
+      "epoch": 0.8213333333333334,
+      "grad_norm": 1.089823722507308,
+      "kl": 0.0517578125,
+      "learning_rate": 5.893333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.734375,
+      "reward_std": 0.056917719542980194,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.734375,
+      "step": 616
+    },
+    {
+      "completion_length": 126.90625,
+      "epoch": 0.8226666666666667,
+      "grad_norm": 1.4861801340558967,
+      "kl": 0.07470703125,
+      "learning_rate": 5.886666666666667e-07,
+      "loss": 0.003,
+      "reward": 1.8102679252624512,
+      "reward_std": 0.06867477297782898,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8102678656578064,
+      "step": 617
+    },
+    {
+      "completion_length": 142.0625,
+      "epoch": 0.824,
+      "grad_norm": 1.4722683124872518,
+      "kl": 0.05078125,
+      "learning_rate": 5.879999999999999e-07,
+      "loss": 0.002,
+      "reward": 1.631250023841858,
+      "reward_std": 0.14592358469963074,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.6781250238418579,
+      "step": 618
+    },
+    {
+      "completion_length": 120.015625,
+      "epoch": 0.8253333333333334,
+      "grad_norm": 4.045383750282507,
+      "kl": 0.0634765625,
+      "learning_rate": 5.873333333333334e-07,
+      "loss": 0.0025,
+      "reward": 1.9140625,
+      "reward_std": 0.10056979209184647,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9140625,
+      "step": 619
+    },
+    {
+      "completion_length": 134.265625,
+      "epoch": 0.8266666666666667,
+      "grad_norm": 2.7282059320569814,
+      "kl": 0.058349609375,
+      "learning_rate": 5.866666666666666e-07,
+      "loss": 0.0023,
+      "reward": 1.8497395515441895,
+      "reward_std": 0.13315638899803162,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8653646111488342,
+      "step": 620
+    },
+    {
+      "completion_length": 124.15625,
+      "epoch": 0.828,
+      "grad_norm": 0.6328006568701829,
+      "kl": 0.04833984375,
+      "learning_rate": 5.86e-07,
+      "loss": 0.0019,
+      "reward": 1.9635417461395264,
+      "reward_std": 0.010416664183139801,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9635417461395264,
+      "step": 621
+    },
+    {
+      "completion_length": 130.90625,
+      "epoch": 0.8293333333333334,
+      "grad_norm": 1.5720801095392782,
+      "kl": 0.0546875,
+      "learning_rate": 5.853333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.780989646911621,
+      "reward_std": 0.10421563684940338,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7809896469116211,
+      "step": 622
+    },
+    {
+      "completion_length": 115.078125,
+      "epoch": 0.8306666666666667,
+      "grad_norm": 1.6527497908554898,
+      "kl": 0.034423828125,
+      "learning_rate": 5.846666666666667e-07,
+      "loss": 0.0014,
+      "reward": 1.7200521230697632,
+      "reward_std": 0.11241096258163452,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7200520634651184,
+      "step": 623
+    },
+    {
+      "completion_length": 137.796875,
+      "epoch": 0.832,
+      "grad_norm": 3.105548641579481,
+      "kl": 0.08837890625,
+      "learning_rate": 5.839999999999999e-07,
+      "loss": 0.0035,
+      "reward": 1.7994792461395264,
+      "reward_std": 0.13970673084259033,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8151041269302368,
+      "step": 624
+    },
+    {
+      "completion_length": 143.4375,
+      "epoch": 0.8333333333333334,
+      "grad_norm": 2.5454874614118754,
+      "kl": 0.056396484375,
+      "learning_rate": 5.833333333333334e-07,
+      "loss": 0.0023,
+      "reward": 1.7218749523162842,
+      "reward_std": 0.22345414757728577,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.784375011920929,
+      "step": 625
+    },
+    {
+      "completion_length": 116.609375,
+      "epoch": 0.8346666666666667,
+      "grad_norm": 1.3840056982368423,
+      "kl": 0.05126953125,
+      "learning_rate": 5.826666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.9166667461395264,
+      "reward_std": 0.07936251908540726,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9166666865348816,
+      "step": 626
+    },
+    {
+      "completion_length": 123.421875,
+      "epoch": 0.836,
+      "grad_norm": 1.2666392460088949,
+      "kl": 0.0361328125,
+      "learning_rate": 5.819999999999999e-07,
+      "loss": 0.0014,
+      "reward": 1.796875,
+      "reward_std": 0.15842358767986298,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.828125,
+      "step": 627
+    },
+    {
+      "completion_length": 126.171875,
+      "epoch": 0.8373333333333334,
+      "grad_norm": 1.6420300430251635,
+      "kl": 0.038330078125,
+      "learning_rate": 5.813333333333334e-07,
+      "loss": 0.0015,
+      "reward": 1.8997396230697632,
+      "reward_std": 0.08717501908540726,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8997396230697632,
+      "step": 628
+    },
+    {
+      "completion_length": 133.140625,
+      "epoch": 0.8386666666666667,
+      "grad_norm": 2.7583125881298094,
+      "kl": 0.0537109375,
+      "learning_rate": 5.806666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.735119104385376,
+      "reward_std": 0.10384036600589752,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.735119104385376,
+      "step": 629
+    },
+    {
+      "completion_length": 130.078125,
+      "epoch": 0.84,
+      "grad_norm": 2.1451962354089034,
+      "kl": 0.0576171875,
+      "learning_rate": 5.8e-07,
+      "loss": 0.0023,
+      "reward": 1.7924479246139526,
+      "reward_std": 0.0947585254907608,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7924479842185974,
+      "step": 630
+    },
+    {
+      "completion_length": 120.796875,
+      "epoch": 0.8413333333333334,
+      "grad_norm": 5.664640352082096,
+      "kl": 0.052490234375,
+      "learning_rate": 5.793333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.8463542461395264,
+      "reward_std": 0.09939011931419373,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8463541865348816,
+      "step": 631
+    },
+    {
+      "completion_length": 130.5625,
+      "epoch": 0.8426666666666667,
+      "grad_norm": 1.3319863054697836,
+      "kl": 0.0556640625,
+      "learning_rate": 5.786666666666667e-07,
+      "loss": 0.0022,
+      "reward": 1.8824561834335327,
+      "reward_std": 0.0961543396115303,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8980811238288879,
+      "step": 632
+    },
+    {
+      "completion_length": 124.078125,
+      "epoch": 0.844,
+      "grad_norm": 0.8675763521349859,
+      "kl": 0.0458984375,
+      "learning_rate": 5.779999999999999e-07,
+      "loss": 0.0018,
+      "reward": 1.8893229961395264,
+      "reward_std": 0.09369336068630219,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9049479365348816,
+      "step": 633
+    },
+    {
+      "completion_length": 123.75,
+      "epoch": 0.8453333333333334,
+      "grad_norm": 2.616774318624436,
+      "kl": 0.068359375,
+      "learning_rate": 5.773333333333334e-07,
+      "loss": 0.0027,
+      "reward": 1.8807291984558105,
+      "reward_std": 0.14369286596775055,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8963541984558105,
+      "step": 634
+    },
+    {
+      "completion_length": 127.328125,
+      "epoch": 0.8466666666666667,
+      "grad_norm": 3.542329779774843,
+      "kl": 0.055419921875,
+      "learning_rate": 5.766666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.7859375476837158,
+      "reward_std": 0.08784337341785431,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.801562488079071,
+      "step": 635
+    },
+    {
+      "completion_length": 131.578125,
+      "epoch": 0.848,
+      "grad_norm": 2.4449487151800837,
+      "kl": 0.043701171875,
+      "learning_rate": 5.76e-07,
+      "loss": 0.0017,
+      "reward": 1.7398438453674316,
+      "reward_std": 0.19949236512184143,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.8023437261581421,
+      "step": 636
+    },
+    {
+      "completion_length": 125.421875,
+      "epoch": 0.8493333333333334,
+      "grad_norm": 2.060263736738446,
+      "kl": 0.061279296875,
+      "learning_rate": 5.753333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.7449777126312256,
+      "reward_std": 0.11625455319881439,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7449777126312256,
+      "step": 637
+    },
+    {
+      "completion_length": 122.484375,
+      "epoch": 0.8506666666666667,
+      "grad_norm": 1.3641639159406238,
+      "kl": 0.062255859375,
+      "learning_rate": 5.746666666666667e-07,
+      "loss": 0.0025,
+      "reward": 1.8630952835083008,
+      "reward_std": 0.04047619178891182,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8630951642990112,
+      "step": 638
+    },
+    {
+      "completion_length": 123.40625,
+      "epoch": 0.852,
+      "grad_norm": 1.7064757920332398,
+      "kl": 0.057373046875,
+      "learning_rate": 5.739999999999999e-07,
+      "loss": 0.0023,
+      "reward": 1.8489583730697632,
+      "reward_std": 0.07452812790870667,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8489582538604736,
+      "step": 639
+    },
+    {
+      "completion_length": 137.6875,
+      "epoch": 0.8533333333333334,
+      "grad_norm": 1.9150804049461518,
+      "kl": 0.036865234375,
+      "learning_rate": 5.733333333333334e-07,
+      "loss": 0.0015,
+      "reward": 1.84765625,
+      "reward_std": 0.11160522699356079,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.86328125,
+      "step": 640
+    },
+    {
+      "completion_length": 129.28125,
+      "epoch": 0.8546666666666667,
+      "grad_norm": 3.027671813848207,
+      "kl": 0.0693359375,
+      "learning_rate": 5.726666666666666e-07,
+      "loss": 0.0028,
+      "reward": 1.7877976894378662,
+      "reward_std": 0.1683688461780548,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8034225702285767,
+      "step": 641
+    },
+    {
+      "completion_length": 124.734375,
+      "epoch": 0.856,
+      "grad_norm": 2.1001144172563304,
+      "kl": 0.059814453125,
+      "learning_rate": 5.719999999999999e-07,
+      "loss": 0.0024,
+      "reward": 1.7239583730697632,
+      "reward_std": 0.08794014900922775,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7239583134651184,
+      "step": 642
+    },
+    {
+      "completion_length": 131.84375,
+      "epoch": 0.8573333333333333,
+      "grad_norm": 3.398481902124305,
+      "kl": 0.07177734375,
+      "learning_rate": 5.713333333333333e-07,
+      "loss": 0.0029,
+      "reward": 1.8776042461395264,
+      "reward_std": 0.11714030802249908,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8776041865348816,
+      "step": 643
+    },
+    {
+      "completion_length": 123.765625,
+      "epoch": 0.8586666666666667,
+      "grad_norm": 4.077531816022191,
+      "kl": 0.0654296875,
+      "learning_rate": 5.706666666666666e-07,
+      "loss": 0.0026,
+      "reward": 1.92578125,
+      "reward_std": 0.07983555644750595,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9257813096046448,
+      "step": 644
+    },
+    {
+      "completion_length": 124.75,
+      "epoch": 0.86,
+      "grad_norm": 2.0639792175876597,
+      "kl": 0.04931640625,
+      "learning_rate": 5.699999999999999e-07,
+      "loss": 0.002,
+      "reward": 1.8291666507720947,
+      "reward_std": 0.05601406842470169,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8291667103767395,
+      "step": 645
+    },
+    {
+      "completion_length": 129.390625,
+      "epoch": 0.8613333333333333,
+      "grad_norm": 1.1360406040587492,
+      "kl": 0.056640625,
+      "learning_rate": 5.693333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.8229167461395264,
+      "reward_std": 0.10634025186300278,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8541666865348816,
+      "step": 646
+    },
+    {
+      "completion_length": 133.15625,
+      "epoch": 0.8626666666666667,
+      "grad_norm": 2.3699052861491676,
+      "kl": 0.0546875,
+      "learning_rate": 5.686666666666667e-07,
+      "loss": 0.0022,
+      "reward": 1.8739583492279053,
+      "reward_std": 0.034944791346788406,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8739583492279053,
+      "step": 647
+    },
+    {
+      "completion_length": 121.328125,
+      "epoch": 0.864,
+      "grad_norm": 1.7366412820998516,
+      "kl": 0.0478515625,
+      "learning_rate": 5.679999999999999e-07,
+      "loss": 0.0019,
+      "reward": 1.890625,
+      "reward_std": 0.09005266427993774,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.890625,
+      "step": 648
+    },
+    {
+      "completion_length": 121.1875,
+      "epoch": 0.8653333333333333,
+      "grad_norm": 1.3984933220232114,
+      "kl": 0.034912109375,
+      "learning_rate": 5.673333333333334e-07,
+      "loss": 0.0014,
+      "reward": 1.734375,
+      "reward_std": 0.08054219186306,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.734375,
+      "step": 649
+    },
+    {
+      "completion_length": 132.859375,
+      "epoch": 0.8666666666666667,
+      "grad_norm": 2.362384309323657,
+      "kl": 0.0634765625,
+      "learning_rate": 5.666666666666666e-07,
+      "loss": 0.0025,
+      "reward": 1.9130208492279053,
+      "reward_std": 0.09170429408550262,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9130208492279053,
+      "step": 650
+    },
+    {
+      "completion_length": 131.96875,
+      "epoch": 0.868,
+      "grad_norm": 2.370568601953114,
+      "kl": 0.08251953125,
+      "learning_rate": 5.66e-07,
+      "loss": 0.0033,
+      "reward": 1.7684895992279053,
+      "reward_std": 0.09858912229537964,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7684895992279053,
+      "step": 651
+    },
+    {
+      "completion_length": 132.65625,
+      "epoch": 0.8693333333333333,
+      "grad_norm": 1.4905137514752813,
+      "kl": 0.036865234375,
+      "learning_rate": 5.653333333333333e-07,
+      "loss": 0.0015,
+      "reward": 1.85546875,
+      "reward_std": 0.11058359593153,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.88671875,
+      "step": 652
+    },
+    {
+      "completion_length": 131.328125,
+      "epoch": 0.8706666666666667,
+      "grad_norm": 1.566785535216872,
+      "kl": 0.06494140625,
+      "learning_rate": 5.646666666666667e-07,
+      "loss": 0.0026,
+      "reward": 1.8718750476837158,
+      "reward_std": 0.13182517886161804,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8875000476837158,
+      "step": 653
+    },
+    {
+      "completion_length": 133.484375,
+      "epoch": 0.872,
+      "grad_norm": 1.444526705580143,
+      "kl": 0.06494140625,
+      "learning_rate": 5.639999999999999e-07,
+      "loss": 0.0026,
+      "reward": 1.8046875,
+      "reward_std": 0.13394904136657715,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.8828125,
+      "step": 654
+    },
+    {
+      "completion_length": 121.15625,
+      "epoch": 0.8733333333333333,
+      "grad_norm": 1.1832884968631492,
+      "kl": 0.0361328125,
+      "learning_rate": 5.633333333333334e-07,
+      "loss": 0.0014,
+      "reward": 1.793229103088379,
+      "reward_std": 0.05248497426509857,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7932291030883789,
+      "step": 655
+    },
+    {
+      "completion_length": 125.296875,
+      "epoch": 0.8746666666666667,
+      "grad_norm": 1.3646182820190396,
+      "kl": 0.045654296875,
+      "learning_rate": 5.626666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.8098958730697632,
+      "reward_std": 0.0552058070898056,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8098958134651184,
+      "step": 656
+    },
+    {
+      "completion_length": 121.015625,
+      "epoch": 0.876,
+      "grad_norm": 1.2924206051354539,
+      "kl": 0.04345703125,
+      "learning_rate": 5.620000000000001e-07,
+      "loss": 0.0017,
+      "reward": 1.946874976158142,
+      "reward_std": 0.010416664183139801,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9468750357627869,
+      "step": 657
+    },
+    {
+      "completion_length": 123.5625,
+      "epoch": 0.8773333333333333,
+      "grad_norm": 0.7508348578835555,
+      "kl": 0.031982421875,
+      "learning_rate": 5.613333333333333e-07,
+      "loss": 0.0013,
+      "reward": 1.9526041746139526,
+      "reward_std": 0.0757172703742981,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9682291746139526,
+      "step": 658
+    },
+    {
+      "completion_length": 121.875,
+      "epoch": 0.8786666666666667,
+      "grad_norm": 1.479586493705865,
+      "kl": 0.03173828125,
+      "learning_rate": 5.606666666666666e-07,
+      "loss": 0.0013,
+      "reward": 1.92578125,
+      "reward_std": 0.09077189117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.92578125,
+      "step": 659
+    },
+    {
+      "completion_length": 116.640625,
+      "epoch": 0.88,
+      "grad_norm": 1.9924764896515237,
+      "kl": 0.044189453125,
+      "learning_rate": 5.6e-07,
+      "loss": 0.0018,
+      "reward": 1.7239582538604736,
+      "reward_std": 0.08156382292509079,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7239583134651184,
+      "step": 660
+    },
+    {
+      "completion_length": 120.953125,
+      "epoch": 0.8813333333333333,
+      "grad_norm": 5.9833460105765335,
+      "kl": 0.0703125,
+      "learning_rate": 5.593333333333333e-07,
+      "loss": 0.0028,
+      "reward": 1.87890625,
+      "reward_std": 0.11035791039466858,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.87890625,
+      "step": 661
+    },
+    {
+      "completion_length": 118.90625,
+      "epoch": 0.8826666666666667,
+      "grad_norm": 1.9098681029215907,
+      "kl": 0.04150390625,
+      "learning_rate": 5.586666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.8463541269302368,
+      "reward_std": 0.036458320915699005,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8463541269302368,
+      "step": 662
+    },
+    {
+      "completion_length": 125.96875,
+      "epoch": 0.884,
+      "grad_norm": 3.081711665750823,
+      "kl": 0.0595703125,
+      "learning_rate": 5.58e-07,
+      "loss": 0.0024,
+      "reward": 1.869531273841858,
+      "reward_std": 0.16113629937171936,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.9007812738418579,
+      "step": 663
+    },
+    {
+      "completion_length": 121.34375,
+      "epoch": 0.8853333333333333,
+      "grad_norm": 1.5205987319797982,
+      "kl": 0.056396484375,
+      "learning_rate": 5.573333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.9119791984558105,
+      "reward_std": 0.0486907958984375,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9119791984558105,
+      "step": 664
+    },
+    {
+      "completion_length": 126.453125,
+      "epoch": 0.8866666666666667,
+      "grad_norm": 3.3264588390435192,
+      "kl": 0.0498046875,
+      "learning_rate": 5.566666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.814843773841858,
+      "reward_std": 0.10841700434684753,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8304687738418579,
+      "step": 665
+    },
+    {
+      "completion_length": 121.421875,
+      "epoch": 0.888,
+      "grad_norm": 1.4515581372434145,
+      "kl": 0.045166015625,
+      "learning_rate": 5.560000000000001e-07,
+      "loss": 0.0018,
+      "reward": 1.816666603088379,
+      "reward_std": 0.15098945796489716,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8166666626930237,
+      "step": 666
+    },
+    {
+      "completion_length": 129.890625,
+      "epoch": 0.8893333333333333,
+      "grad_norm": 2.9664501685718223,
+      "kl": 0.0576171875,
+      "learning_rate": 5.553333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.7485119104385376,
+      "reward_std": 0.12379857152700424,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7797619104385376,
+      "step": 667
+    },
+    {
+      "completion_length": 124.265625,
+      "epoch": 0.8906666666666667,
+      "grad_norm": 1.5161659600901773,
+      "kl": 0.048095703125,
+      "learning_rate": 5.546666666666667e-07,
+      "loss": 0.0019,
+      "reward": 1.8197916746139526,
+      "reward_std": 0.16842862963676453,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8510416746139526,
+      "step": 668
+    },
+    {
+      "completion_length": 118.15625,
+      "epoch": 0.892,
+      "grad_norm": 1.0782845212267462,
+      "kl": 0.05419921875,
+      "learning_rate": 5.54e-07,
+      "loss": 0.0022,
+      "reward": 1.8046875,
+      "reward_std": 0.015625,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8046875,
+      "step": 669
+    },
+    {
+      "completion_length": 134.0625,
+      "epoch": 0.8933333333333333,
+      "grad_norm": 1.2931134913473565,
+      "kl": 0.032958984375,
+      "learning_rate": 5.533333333333334e-07,
+      "loss": 0.0013,
+      "reward": 1.8229167461395264,
+      "reward_std": 0.14229430258274078,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8697916865348816,
+      "step": 670
+    },
+    {
+      "completion_length": 117.59375,
+      "epoch": 0.8946666666666667,
+      "grad_norm": 2.3614805602797464,
+      "kl": 0.0576171875,
+      "learning_rate": 5.526666666666666e-07,
+      "loss": 0.0023,
+      "reward": 1.8111979961395264,
+      "reward_std": 0.1513664871454239,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8268228769302368,
+      "step": 671
+    },
+    {
+      "completion_length": 125.359375,
+      "epoch": 0.896,
+      "grad_norm": 1.5281804723268595,
+      "kl": 0.044677734375,
+      "learning_rate": 5.520000000000001e-07,
+      "loss": 0.0018,
+      "reward": 1.7565104961395264,
+      "reward_std": 0.07807163894176483,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8033854365348816,
+      "step": 672
+    },
+    {
+      "completion_length": 119.140625,
+      "epoch": 0.8973333333333333,
+      "grad_norm": 1.797791587850255,
+      "kl": 0.0294189453125,
+      "learning_rate": 5.513333333333333e-07,
+      "loss": 0.0012,
+      "reward": 1.84375,
+      "reward_std": 0.16108438372612,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.859375,
+      "step": 673
+    },
+    {
+      "completion_length": 110.984375,
+      "epoch": 0.8986666666666666,
+      "grad_norm": 2.620675075347278,
+      "kl": 0.059326171875,
+      "learning_rate": 5.506666666666666e-07,
+      "loss": 0.0024,
+      "reward": 1.7526042461395264,
+      "reward_std": 0.05409187823534012,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7526041865348816,
+      "step": 674
+    },
+    {
+      "completion_length": 108.71875,
+      "epoch": 0.9,
+      "grad_norm": 3.135479159271414,
+      "kl": 0.05322265625,
+      "learning_rate": 5.5e-07,
+      "loss": 0.0021,
+      "reward": 1.8958333730697632,
+      "reward_std": 0.1402510553598404,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8958333730697632,
+      "step": 675
+    },
+    {
+      "completion_length": 122.234375,
+      "epoch": 0.9013333333333333,
+      "grad_norm": 1.7987777336831978,
+      "kl": 0.04150390625,
+      "learning_rate": 5.493333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.890885353088379,
+      "reward_std": 0.07275882363319397,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9065104722976685,
+      "step": 676
+    },
+    {
+      "completion_length": 113.484375,
+      "epoch": 0.9026666666666666,
+      "grad_norm": 3.1791706489752656,
+      "kl": 0.046630859375,
+      "learning_rate": 5.486666666666666e-07,
+      "loss": 0.0019,
+      "reward": 1.723177194595337,
+      "reward_std": 0.1253332495689392,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7231771349906921,
+      "step": 677
+    },
+    {
+      "completion_length": 112.90625,
+      "epoch": 0.904,
+      "grad_norm": 1.3789042259464892,
+      "kl": 0.076171875,
+      "learning_rate": 5.48e-07,
+      "loss": 0.003,
+      "reward": 1.8739583492279053,
+      "reward_std": 0.12307557463645935,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.87395840883255,
+      "step": 678
+    },
+    {
+      "completion_length": 123.3125,
+      "epoch": 0.9053333333333333,
+      "grad_norm": 2.3345465063071265,
+      "kl": 0.0546875,
+      "learning_rate": 5.473333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.776836633682251,
+      "reward_std": 0.16780467331409454,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.792461633682251,
+      "step": 679
+    },
+    {
+      "completion_length": 113.3125,
+      "epoch": 0.9066666666666666,
+      "grad_norm": 1.7191782808635414,
+      "kl": 0.07177734375,
+      "learning_rate": 5.466666666666666e-07,
+      "loss": 0.0029,
+      "reward": 1.8541667461395264,
+      "reward_std": 0.08863871544599533,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8541666865348816,
+      "step": 680
+    },
+    {
+      "completion_length": 106.515625,
+      "epoch": 0.908,
+      "grad_norm": 1.8882504673520515,
+      "kl": 0.0478515625,
+      "learning_rate": 5.46e-07,
+      "loss": 0.0019,
+      "reward": 1.6731771230697632,
+      "reward_std": 0.0546875,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6731771230697632,
+      "step": 681
+    },
+    {
+      "completion_length": 121.5625,
+      "epoch": 0.9093333333333333,
+      "grad_norm": 1.0469650056936501,
+      "kl": 0.033203125,
+      "learning_rate": 5.453333333333333e-07,
+      "loss": 0.0013,
+      "reward": 1.71875,
+      "reward_std": 0.0416666641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.71875,
+      "step": 682
+    },
+    {
+      "completion_length": 121.703125,
+      "epoch": 0.9106666666666666,
+      "grad_norm": 1.776012152871689,
+      "kl": 0.045166015625,
+      "learning_rate": 5.446666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.7890625,
+      "reward_std": 0.1410323530435562,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8203125,
+      "step": 683
+    },
+    {
+      "completion_length": 107.015625,
+      "epoch": 0.912,
+      "grad_norm": 0.9309709944410245,
+      "kl": 0.0361328125,
+      "learning_rate": 5.44e-07,
+      "loss": 0.0014,
+      "reward": 1.921875,
+      "reward_std": 0.0625,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.921875,
+      "step": 684
+    },
+    {
+      "completion_length": 117.46875,
+      "epoch": 0.9133333333333333,
+      "grad_norm": 3.061968659359718,
+      "kl": 0.05810546875,
+      "learning_rate": 5.433333333333334e-07,
+      "loss": 0.0023,
+      "reward": 1.8776042461395264,
+      "reward_std": 0.11098645627498627,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8776041865348816,
+      "step": 685
+    },
+    {
+      "completion_length": 107.71875,
+      "epoch": 0.9146666666666666,
+      "grad_norm": 1.9308565784356362,
+      "kl": 0.05029296875,
+      "learning_rate": 5.426666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.9356770515441895,
+      "reward_std": 0.0792156308889389,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9356771111488342,
+      "step": 686
+    },
+    {
+      "completion_length": 113.265625,
+      "epoch": 0.916,
+      "grad_norm": 6.116339636191993,
+      "kl": 0.04833984375,
+      "learning_rate": 5.420000000000001e-07,
+      "loss": 0.0019,
+      "reward": 1.9044270515441895,
+      "reward_std": 0.09520578384399414,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9044270515441895,
+      "step": 687
+    },
+    {
+      "completion_length": 129.59375,
+      "epoch": 0.9173333333333333,
+      "grad_norm": 10.253803746799017,
+      "kl": 0.07568359375,
+      "learning_rate": 5.413333333333333e-07,
+      "loss": 0.003,
+      "reward": 1.7927827835083008,
+      "reward_std": 0.11020498722791672,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.808407723903656,
+      "step": 688
+    },
+    {
+      "completion_length": 112.34375,
+      "epoch": 0.9186666666666666,
+      "grad_norm": 1.5579830631797327,
+      "kl": 0.056640625,
+      "learning_rate": 5.406666666666666e-07,
+      "loss": 0.0023,
+      "reward": 1.8260416984558105,
+      "reward_std": 0.0729166641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8260416388511658,
+      "step": 689
+    },
+    {
+      "completion_length": 107.703125,
+      "epoch": 0.92,
+      "grad_norm": 24.444993245437992,
+      "kl": 0.0546875,
+      "learning_rate": 5.4e-07,
+      "loss": 0.0022,
+      "reward": 1.875,
+      "reward_std": 0.125,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.890625,
+      "step": 690
+    },
+    {
+      "completion_length": 114.640625,
+      "epoch": 0.9213333333333333,
+      "grad_norm": 2.522834943233949,
+      "kl": 0.05517578125,
+      "learning_rate": 5.393333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.91796875,
+      "reward_std": 0.0675213560461998,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.91796875,
+      "step": 691
+    },
+    {
+      "completion_length": 125.875,
+      "epoch": 0.9226666666666666,
+      "grad_norm": 5.009027930663783,
+      "kl": 0.0810546875,
+      "learning_rate": 5.386666666666666e-07,
+      "loss": 0.0032,
+      "reward": 1.7101562023162842,
+      "reward_std": 0.13757210969924927,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7101562023162842,
+      "step": 692
+    },
+    {
+      "completion_length": 120.953125,
+      "epoch": 0.924,
+      "grad_norm": 1.96060251120186,
+      "kl": 0.031005859375,
+      "learning_rate": 5.38e-07,
+      "loss": 0.0012,
+      "reward": 1.871354103088379,
+      "reward_std": 0.1496259570121765,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.9026041626930237,
+      "step": 693
+    },
+    {
+      "completion_length": 122.390625,
+      "epoch": 0.9253333333333333,
+      "grad_norm": 2.065141411295824,
+      "kl": 0.0390625,
+      "learning_rate": 5.373333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.6661458015441895,
+      "reward_std": 0.2166902720928192,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6817708015441895,
+      "step": 694
+    },
+    {
+      "completion_length": 118.3125,
+      "epoch": 0.9266666666666666,
+      "grad_norm": 0.869459152653853,
+      "kl": 0.043212890625,
+      "learning_rate": 5.366666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.7794270515441895,
+      "reward_std": 0.016715625301003456,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7794270515441895,
+      "step": 695
+    },
+    {
+      "completion_length": 123.46875,
+      "epoch": 0.928,
+      "grad_norm": 1.4789617557001045,
+      "kl": 0.06396484375,
+      "learning_rate": 5.36e-07,
+      "loss": 0.0026,
+      "reward": 1.845312476158142,
+      "reward_std": 0.04644883796572685,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8453125357627869,
+      "step": 696
+    },
+    {
+      "completion_length": 111.71875,
+      "epoch": 0.9293333333333333,
+      "grad_norm": 1.7155295274542617,
+      "kl": 0.053466796875,
+      "learning_rate": 5.353333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.859375,
+      "reward_std": 0.0729166641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.859375,
+      "step": 697
+    },
+    {
+      "completion_length": 119.625,
+      "epoch": 0.9306666666666666,
+      "grad_norm": 1.7598815218390844,
+      "kl": 0.05126953125,
+      "learning_rate": 5.346666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.8119791746139526,
+      "reward_std": 0.0151531295850873,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8119791746139526,
+      "step": 698
+    },
+    {
+      "completion_length": 117.125,
+      "epoch": 0.932,
+      "grad_norm": 0.715620992248015,
+      "kl": 0.029541015625,
+      "learning_rate": 5.34e-07,
+      "loss": 0.0012,
+      "reward": 1.8229167461395264,
+      "reward_std": 0.08711542934179306,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8385416269302368,
+      "step": 699
+    },
+    {
+      "completion_length": 128.90625,
+      "epoch": 0.9333333333333333,
+      "grad_norm": 3.0127391519784386,
+      "kl": 0.078125,
+      "learning_rate": 5.333333333333333e-07,
+      "loss": 0.0031,
+      "reward": 1.8570313453674316,
+      "reward_std": 0.06718750298023224,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8726562261581421,
+      "step": 700
+    },
+    {
+      "completion_length": 137.953125,
+      "epoch": 0.9346666666666666,
+      "grad_norm": 1.495111951057385,
+      "kl": 0.0771484375,
+      "learning_rate": 5.326666666666666e-07,
+      "loss": 0.0031,
+      "reward": 1.8307292461395264,
+      "reward_std": 0.11479923129081726,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8307292461395264,
+      "step": 701
+    },
+    {
+      "completion_length": 123.03125,
+      "epoch": 0.936,
+      "grad_norm": 1.4725968165373309,
+      "kl": 0.0478515625,
+      "learning_rate": 5.32e-07,
+      "loss": 0.0019,
+      "reward": 1.9276041984558105,
+      "reward_std": 0.028007034212350845,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9276041984558105,
+      "step": 702
+    },
+    {
+      "completion_length": 135.890625,
+      "epoch": 0.9373333333333334,
+      "grad_norm": 1.6590177372496888,
+      "kl": 0.058837890625,
+      "learning_rate": 5.313333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.851302146911621,
+      "reward_std": 0.1721620261669159,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8825520277023315,
+      "step": 703
+    },
+    {
+      "completion_length": 120.109375,
+      "epoch": 0.9386666666666666,
+      "grad_norm": 1.8999936014123127,
+      "kl": 0.04931640625,
+      "learning_rate": 5.306666666666665e-07,
+      "loss": 0.002,
+      "reward": 1.9281994104385376,
+      "reward_std": 0.09881462901830673,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9281994104385376,
+      "step": 704
+    },
+    {
+      "completion_length": 121.359375,
+      "epoch": 0.94,
+      "grad_norm": 1.6175375610456781,
+      "kl": 0.03857421875,
+      "learning_rate": 5.3e-07,
+      "loss": 0.0015,
+      "reward": 1.817968726158142,
+      "reward_std": 0.05743855983018875,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8179687261581421,
+      "step": 705
+    },
+    {
+      "completion_length": 121.71875,
+      "epoch": 0.9413333333333334,
+      "grad_norm": 1.4434396743452034,
+      "kl": 0.042724609375,
+      "learning_rate": 5.293333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.91015625,
+      "reward_std": 0.0661257952451706,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.91015625,
+      "step": 706
+    },
+    {
+      "completion_length": 129.6875,
+      "epoch": 0.9426666666666667,
+      "grad_norm": 0.7203039509317434,
+      "kl": 0.05224609375,
+      "learning_rate": 5.286666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.8385417461395264,
+      "reward_std": 0.08228103816509247,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8541666865348816,
+      "step": 707
+    },
+    {
+      "completion_length": 122.953125,
+      "epoch": 0.944,
+      "grad_norm": 0.8512965021368486,
+      "kl": 0.059814453125,
+      "learning_rate": 5.28e-07,
+      "loss": 0.0024,
+      "reward": 1.8072917461395264,
+      "reward_std": 0.0416666641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8072916865348816,
+      "step": 708
+    },
+    {
+      "completion_length": 115.515625,
+      "epoch": 0.9453333333333334,
+      "grad_norm": 2.8290766173120256,
+      "kl": 0.076171875,
+      "learning_rate": 5.273333333333333e-07,
+      "loss": 0.0031,
+      "reward": 1.783593773841858,
+      "reward_std": 0.08660522103309631,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7835937738418579,
+      "step": 709
+    },
+    {
+      "completion_length": 128.1875,
+      "epoch": 0.9466666666666667,
+      "grad_norm": 1.2154518473514615,
+      "kl": 0.0478515625,
+      "learning_rate": 5.266666666666666e-07,
+      "loss": 0.0019,
+      "reward": 1.7375000715255737,
+      "reward_std": 0.08237498998641968,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.768750011920929,
+      "step": 710
+    },
+    {
+      "completion_length": 118.578125,
+      "epoch": 0.948,
+      "grad_norm": 4.199619244139667,
+      "kl": 0.053955078125,
+      "learning_rate": 5.26e-07,
+      "loss": 0.0022,
+      "reward": 1.8682291507720947,
+      "reward_std": 0.10165463387966156,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8682292103767395,
+      "step": 711
+    },
+    {
+      "completion_length": 127.875,
+      "epoch": 0.9493333333333334,
+      "grad_norm": 1.88380008638717,
+      "kl": 0.06640625,
+      "learning_rate": 5.253333333333333e-07,
+      "loss": 0.0027,
+      "reward": 1.7434896230697632,
+      "reward_std": 0.0807291641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7434895634651184,
+      "step": 712
+    },
+    {
+      "completion_length": 124.578125,
+      "epoch": 0.9506666666666667,
+      "grad_norm": 7.817445143753372,
+      "kl": 0.08349609375,
+      "learning_rate": 5.246666666666666e-07,
+      "loss": 0.0033,
+      "reward": 1.895052194595337,
+      "reward_std": 0.0703125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8950521349906921,
+      "step": 713
+    },
+    {
+      "completion_length": 121.125,
+      "epoch": 0.952,
+      "grad_norm": 41.81824517937354,
+      "kl": 0.045654296875,
+      "learning_rate": 5.24e-07,
+      "loss": 0.0018,
+      "reward": 1.894270896911621,
+      "reward_std": 0.04479166492819786,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8942708969116211,
+      "step": 714
+    },
+    {
+      "completion_length": 117.65625,
+      "epoch": 0.9533333333333334,
+      "grad_norm": 2.1821864869487246,
+      "kl": 0.060546875,
+      "learning_rate": 5.233333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.8653645515441895,
+      "reward_std": 0.08000297844409943,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8653646111488342,
+      "step": 715
+    },
+    {
+      "completion_length": 138.34375,
+      "epoch": 0.9546666666666667,
+      "grad_norm": 1.1555574377413018,
+      "kl": 0.04248046875,
+      "learning_rate": 5.226666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.8406250476837158,
+      "reward_std": 0.0973391979932785,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.871874988079071,
+      "step": 716
+    },
+    {
+      "completion_length": 122.75,
+      "epoch": 0.956,
+      "grad_norm": 1.5054139457686722,
+      "kl": 0.07861328125,
+      "learning_rate": 5.22e-07,
+      "loss": 0.0031,
+      "reward": 1.845238208770752,
+      "reward_std": 0.05305049568414688,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8452380895614624,
+      "step": 717
+    },
+    {
+      "completion_length": 127.578125,
+      "epoch": 0.9573333333333334,
+      "grad_norm": 2.7868944295517415,
+      "kl": 0.0498046875,
+      "learning_rate": 5.213333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.7239583730697632,
+      "reward_std": 0.09418178349733353,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7239583730697632,
+      "step": 718
+    },
+    {
+      "completion_length": 122.6875,
+      "epoch": 0.9586666666666667,
+      "grad_norm": 0.7319511231082507,
+      "kl": 0.051025390625,
+      "learning_rate": 5.206666666666667e-07,
+      "loss": 0.002,
+      "reward": 1.9557292461395264,
+      "reward_std": 0.009973200969398022,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9557291865348816,
+      "step": 719
+    },
+    {
+      "completion_length": 126.625,
+      "epoch": 0.96,
+      "grad_norm": 1.5772687640493568,
+      "kl": 0.05126953125,
+      "learning_rate": 5.2e-07,
+      "loss": 0.0021,
+      "reward": 1.7635416984558105,
+      "reward_std": 0.21521097421646118,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7947916984558105,
+      "step": 720
+    },
+    {
+      "completion_length": 120.046875,
+      "epoch": 0.9613333333333334,
+      "grad_norm": 3.589301394976807,
+      "kl": 0.0517578125,
+      "learning_rate": 5.193333333333332e-07,
+      "loss": 0.0021,
+      "reward": 1.8270833492279053,
+      "reward_std": 0.1205500140786171,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8270833492279053,
+      "step": 721
+    },
+    {
+      "completion_length": 126.171875,
+      "epoch": 0.9626666666666667,
+      "grad_norm": 1.2263526454998366,
+      "kl": 0.055419921875,
+      "learning_rate": 5.186666666666667e-07,
+      "loss": 0.0022,
+      "reward": 1.863802194595337,
+      "reward_std": 0.06084674596786499,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8638020753860474,
+      "step": 722
+    },
+    {
+      "completion_length": 131.625,
+      "epoch": 0.964,
+      "grad_norm": 1.2824548761887817,
+      "kl": 0.04736328125,
+      "learning_rate": 5.18e-07,
+      "loss": 0.0019,
+      "reward": 1.8327217102050781,
+      "reward_std": 0.11001384258270264,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8483467698097229,
+      "step": 723
+    },
+    {
+      "completion_length": 122.671875,
+      "epoch": 0.9653333333333334,
+      "grad_norm": 1.3801754738165446,
+      "kl": 0.038818359375,
+      "learning_rate": 5.173333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.922619104385376,
+      "reward_std": 0.08304721862077713,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.922619104385376,
+      "step": 724
+    },
+    {
+      "completion_length": 126.125,
+      "epoch": 0.9666666666666667,
+      "grad_norm": 1.3667983297699517,
+      "kl": 0.0546875,
+      "learning_rate": 5.166666666666667e-07,
+      "loss": 0.0022,
+      "reward": 1.9249999523162842,
+      "reward_std": 0.018278129398822784,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9250000715255737,
+      "step": 725
+    },
+    {
+      "completion_length": 128.703125,
+      "epoch": 0.968,
+      "grad_norm": 0.7689549409659756,
+      "kl": 0.026611328125,
+      "learning_rate": 5.16e-07,
+      "loss": 0.0011,
+      "reward": 1.9166666269302368,
+      "reward_std": 0.03726406767964363,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9166666269302368,
+      "step": 726
+    },
+    {
+      "completion_length": 139.4375,
+      "epoch": 0.9693333333333334,
+      "grad_norm": 1.0419404563084698,
+      "kl": 0.04541015625,
+      "learning_rate": 5.153333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.8020832538604736,
+      "reward_std": 0.09976407140493393,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8333333730697632,
+      "step": 727
+    },
+    {
+      "completion_length": 118.09375,
+      "epoch": 0.9706666666666667,
+      "grad_norm": 1.928027945640226,
+      "kl": 0.049560546875,
+      "learning_rate": 5.146666666666667e-07,
+      "loss": 0.002,
+      "reward": 1.9270833730697632,
+      "reward_std": 0.043278127908706665,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9270833730697632,
+      "step": 728
+    },
+    {
+      "completion_length": 129.5,
+      "epoch": 0.972,
+      "grad_norm": 2.2664448352905353,
+      "kl": 0.06884765625,
+      "learning_rate": 5.14e-07,
+      "loss": 0.0027,
+      "reward": 1.7807292938232422,
+      "reward_std": 0.050822507590055466,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7807291746139526,
+      "step": 729
+    },
+    {
+      "completion_length": 140.265625,
+      "epoch": 0.9733333333333334,
+      "grad_norm": 1.6739439987712303,
+      "kl": 0.060302734375,
+      "learning_rate": 5.133333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.811532735824585,
+      "reward_std": 0.1380189061164856,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.827157735824585,
+      "step": 730
+    },
+    {
+      "completion_length": 138.78125,
+      "epoch": 0.9746666666666667,
+      "grad_norm": 2.6039806062291633,
+      "kl": 0.060302734375,
+      "learning_rate": 5.126666666666667e-07,
+      "loss": 0.0024,
+      "reward": 1.856696367263794,
+      "reward_std": 0.1437486708164215,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8566964268684387,
+      "step": 731
+    },
+    {
+      "completion_length": 121.09375,
+      "epoch": 0.976,
+      "grad_norm": 1.6121627852433495,
+      "kl": 0.05078125,
+      "learning_rate": 5.12e-07,
+      "loss": 0.002,
+      "reward": 1.9177827835083008,
+      "reward_std": 0.058675527572631836,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9177827835083008,
+      "step": 732
+    },
+    {
+      "completion_length": 124.703125,
+      "epoch": 0.9773333333333334,
+      "grad_norm": 2.6736154300386006,
+      "kl": 0.047607421875,
+      "learning_rate": 5.113333333333333e-07,
+      "loss": 0.0019,
+      "reward": 1.8145833015441895,
+      "reward_std": 0.07253463566303253,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8145833611488342,
+      "step": 733
+    },
+    {
+      "completion_length": 131.78125,
+      "epoch": 0.9786666666666667,
+      "grad_norm": 1.7096020900210707,
+      "kl": 0.034423828125,
+      "learning_rate": 5.106666666666667e-07,
+      "loss": 0.0014,
+      "reward": 1.671875,
+      "reward_std": 0.22321045398712158,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7031249403953552,
+      "step": 734
+    },
+    {
+      "completion_length": 132.265625,
+      "epoch": 0.98,
+      "grad_norm": 0.9672888389285097,
+      "kl": 0.052978515625,
+      "learning_rate": 5.1e-07,
+      "loss": 0.0021,
+      "reward": 1.7218750715255737,
+      "reward_std": 0.09710326790809631,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7531249523162842,
+      "step": 735
+    },
+    {
+      "completion_length": 114.96875,
+      "epoch": 0.9813333333333333,
+      "grad_norm": 2.5602153303413377,
+      "kl": 0.042236328125,
+      "learning_rate": 5.093333333333332e-07,
+      "loss": 0.0017,
+      "reward": 1.9317708015441895,
+      "reward_std": 0.031683363020420074,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9317708611488342,
+      "step": 736
+    },
+    {
+      "completion_length": 119.375,
+      "epoch": 0.9826666666666667,
+      "grad_norm": 1.7239755222451074,
+      "kl": 0.055908203125,
+      "learning_rate": 5.086666666666667e-07,
+      "loss": 0.0022,
+      "reward": 1.9135416746139526,
+      "reward_std": 0.10096687078475952,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9135416746139526,
+      "step": 737
+    },
+    {
+      "completion_length": 132.921875,
+      "epoch": 0.984,
+      "grad_norm": 1.6017390759750405,
+      "kl": 0.0634765625,
+      "learning_rate": 5.079999999999999e-07,
+      "loss": 0.0025,
+      "reward": 1.7580729722976685,
+      "reward_std": 0.0986909493803978,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7580728530883789,
+      "step": 738
+    },
+    {
+      "completion_length": 125.859375,
+      "epoch": 0.9853333333333333,
+      "grad_norm": 1.2966419368170272,
+      "kl": 0.036376953125,
+      "learning_rate": 5.073333333333333e-07,
+      "loss": 0.0015,
+      "reward": 1.8914062976837158,
+      "reward_std": 0.03764688968658447,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.891406238079071,
+      "step": 739
+    },
+    {
+      "completion_length": 126.640625,
+      "epoch": 0.9866666666666667,
+      "grad_norm": 0.9239858344706915,
+      "kl": 0.05908203125,
+      "learning_rate": 5.066666666666667e-07,
+      "loss": 0.0024,
+      "reward": 1.8156249523162842,
+      "reward_std": 0.09375,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.815625011920929,
+      "step": 740
+    },
+    {
+      "completion_length": 123.734375,
+      "epoch": 0.988,
+      "grad_norm": 1.8350963246149592,
+      "kl": 0.054931640625,
+      "learning_rate": 5.06e-07,
+      "loss": 0.0022,
+      "reward": 1.8440476655960083,
+      "reward_std": 0.07665009796619415,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8440476655960083,
+      "step": 741
+    },
+    {
+      "completion_length": 117.234375,
+      "epoch": 0.9893333333333333,
+      "grad_norm": 1.5756165954741763,
+      "kl": 0.055908203125,
+      "learning_rate": 5.053333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.8718750476837158,
+      "reward_std": 0.08400105684995651,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.887499988079071,
+      "step": 742
+    },
+    {
+      "completion_length": 121.4375,
+      "epoch": 0.9906666666666667,
+      "grad_norm": 1.762510041841452,
+      "kl": 0.054931640625,
+      "learning_rate": 5.046666666666667e-07,
+      "loss": 0.0022,
+      "reward": 1.7846354246139526,
+      "reward_std": 0.09342250972986221,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8002604246139526,
+      "step": 743
+    },
+    {
+      "completion_length": 120.0625,
+      "epoch": 0.992,
+      "grad_norm": 1.7066049614271688,
+      "kl": 0.036376953125,
+      "learning_rate": 5.04e-07,
+      "loss": 0.0015,
+      "reward": 1.703385353088379,
+      "reward_std": 0.13973021507263184,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7033854722976685,
+      "step": 744
+    },
+    {
+      "completion_length": 122.625,
+      "epoch": 0.9933333333333333,
+      "grad_norm": 2.751912502938023,
+      "kl": 0.057373046875,
+      "learning_rate": 5.033333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.9296875,
+      "reward_std": 0.06916174292564392,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9296875,
+      "step": 745
+    },
+    {
+      "completion_length": 134.359375,
+      "epoch": 0.9946666666666667,
+      "grad_norm": 3.119838604689424,
+      "kl": 0.080078125,
+      "learning_rate": 5.026666666666667e-07,
+      "loss": 0.0032,
+      "reward": 1.826562523841858,
+      "reward_std": 0.08806978911161423,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8265625834465027,
+      "step": 746
+    },
+    {
+      "completion_length": 118.875,
+      "epoch": 0.996,
+      "grad_norm": 1.9813392516612531,
+      "kl": 0.036865234375,
+      "learning_rate": 5.02e-07,
+      "loss": 0.0015,
+      "reward": 1.5437500476837158,
+      "reward_std": 0.2105102390050888,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.574999988079071,
+      "step": 747
+    },
+    {
+      "completion_length": 115.28125,
+      "epoch": 0.9973333333333333,
+      "grad_norm": 2.9980049987278847,
+      "kl": 0.0625,
+      "learning_rate": 5.013333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.8610119819641113,
+      "reward_std": 0.04300735518336296,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8610119819641113,
+      "step": 748
+    },
+    {
+      "completion_length": 124.90625,
+      "epoch": 0.9986666666666667,
+      "grad_norm": 1.4310108884444932,
+      "kl": 0.053955078125,
+      "learning_rate": 5.006666666666667e-07,
+      "loss": 0.0022,
+      "reward": 1.8776042461395264,
+      "reward_std": 0.08457084745168686,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8776042461395264,
+      "step": 749
+    },
+    {
+      "completion_length": 126.328125,
+      "epoch": 1.0,
+      "grad_norm": 1.373717915367149,
+      "kl": 0.0361328125,
+      "learning_rate": 5e-07,
+      "loss": 0.0014,
+      "reward": 1.8072916269302368,
+      "reward_std": 0.10459846258163452,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8072916269302368,
+      "step": 750
+    },
+    {
+      "completion_length": 122.828125,
+      "epoch": 1.0013333333333334,
+      "grad_norm": 1.7431411082949504,
+      "kl": 0.033447265625,
+      "learning_rate": 4.993333333333333e-07,
+      "loss": 0.0013,
+      "reward": 1.8914062976837158,
+      "reward_std": 0.12656250596046448,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.891406238079071,
+      "step": 751
+    },
+    {
+      "completion_length": 122.53125,
+      "epoch": 1.0026666666666666,
+      "grad_norm": 3.610330758405405,
+      "kl": 0.0654296875,
+      "learning_rate": 4.986666666666666e-07,
+      "loss": 0.0026,
+      "reward": 1.8541667461395264,
+      "reward_std": 0.07173699140548706,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8541667461395264,
+      "step": 752
+    },
+    {
+      "completion_length": 127.8125,
+      "epoch": 1.004,
+      "grad_norm": 2.5468746029035634,
+      "kl": 0.041748046875,
+      "learning_rate": 4.979999999999999e-07,
+      "loss": 0.0017,
+      "reward": 1.8255208730697632,
+      "reward_std": 0.07560735195875168,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8255208730697632,
+      "step": 753
+    },
+    {
+      "completion_length": 134.5,
+      "epoch": 1.0053333333333334,
+      "grad_norm": 1.4481004908848658,
+      "kl": 0.0537109375,
+      "learning_rate": 4.973333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.8565104007720947,
+      "reward_std": 0.03177490830421448,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8565104007720947,
+      "step": 754
+    },
+    {
+      "completion_length": 148.890625,
+      "epoch": 1.0066666666666666,
+      "grad_norm": 1.7421502200435888,
+      "kl": 0.04931640625,
+      "learning_rate": 4.966666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.679947853088379,
+      "reward_std": 0.23249438405036926,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.7580729722976685,
+      "step": 755
+    },
+    {
+      "completion_length": 124.3125,
+      "epoch": 1.008,
+      "grad_norm": 5.181188959949322,
+      "kl": 0.043212890625,
+      "learning_rate": 4.96e-07,
+      "loss": 0.0017,
+      "reward": 1.9322917461395264,
+      "reward_std": 0.059708863496780396,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9322916269302368,
+      "step": 756
+    },
+    {
+      "completion_length": 137.921875,
+      "epoch": 1.0093333333333334,
+      "grad_norm": 3.2055741067666306,
+      "kl": 0.042236328125,
+      "learning_rate": 4.953333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.9047619104385376,
+      "reward_std": 0.08408564329147339,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9203868508338928,
+      "step": 757
+    },
+    {
+      "completion_length": 112.46875,
+      "epoch": 1.0106666666666666,
+      "grad_norm": 1.4645486236044234,
+      "kl": 0.036376953125,
+      "learning_rate": 4.946666666666666e-07,
+      "loss": 0.0015,
+      "reward": 1.8742187023162842,
+      "reward_std": 0.02135416492819786,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8742187023162842,
+      "step": 758
+    },
+    {
+      "completion_length": 121.8125,
+      "epoch": 1.012,
+      "grad_norm": 1.7401396490270826,
+      "kl": 0.052978515625,
+      "learning_rate": 4.94e-07,
+      "loss": 0.0021,
+      "reward": 1.871354103088379,
+      "reward_std": 0.06405924260616302,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8713542222976685,
+      "step": 759
+    },
+    {
+      "completion_length": 127.375,
+      "epoch": 1.0133333333333334,
+      "grad_norm": 1.554107578617931,
+      "kl": 0.059326171875,
+      "learning_rate": 4.933333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.92578125,
+      "reward_std": 0.0428752601146698,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.92578125,
+      "step": 760
+    },
+    {
+      "completion_length": 130.5,
+      "epoch": 1.0146666666666666,
+      "grad_norm": 1.955383052461807,
+      "kl": 0.09814453125,
+      "learning_rate": 4.926666666666667e-07,
+      "loss": 0.0039,
+      "reward": 1.875,
+      "reward_std": 0.15000000596046448,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.875,
+      "step": 761
+    },
+    {
+      "completion_length": 121.59375,
+      "epoch": 1.016,
+      "grad_norm": 1.08023008263303,
+      "kl": 0.03173828125,
+      "learning_rate": 4.92e-07,
+      "loss": 0.0013,
+      "reward": 1.8541667461395264,
+      "reward_std": 0.09976406395435333,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8541666269302368,
+      "step": 762
+    },
+    {
+      "completion_length": 126.984375,
+      "epoch": 1.0173333333333334,
+      "grad_norm": 2.2591351275594493,
+      "kl": 0.0556640625,
+      "learning_rate": 4.913333333333334e-07,
+      "loss": 0.0022,
+      "reward": 1.6966146230697632,
+      "reward_std": 0.24637296795845032,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6966146230697632,
+      "step": 763
+    },
+    {
+      "completion_length": 127.328125,
+      "epoch": 1.0186666666666666,
+      "grad_norm": 0.5657961237891259,
+      "kl": 0.0634765625,
+      "learning_rate": 4.906666666666666e-07,
+      "loss": 0.0025,
+      "reward": 1.9354166984558105,
+      "reward_std": 0.03608439117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9354166984558105,
+      "step": 764
+    },
+    {
+      "completion_length": 127.03125,
+      "epoch": 1.02,
+      "grad_norm": 1.7507302060800933,
+      "kl": 0.043212890625,
+      "learning_rate": 4.9e-07,
+      "loss": 0.0017,
+      "reward": 1.8111979961395264,
+      "reward_std": 0.11779606342315674,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8268229365348816,
+      "step": 765
+    },
+    {
+      "completion_length": 137.234375,
+      "epoch": 1.0213333333333334,
+      "grad_norm": 1.3510349815614253,
+      "kl": 0.039794921875,
+      "learning_rate": 4.893333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.8958333730697632,
+      "reward_std": 0.036674223840236664,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8958332538604736,
+      "step": 766
+    },
+    {
+      "completion_length": 122.796875,
+      "epoch": 1.0226666666666666,
+      "grad_norm": 1.111924854407035,
+      "kl": 0.05078125,
+      "learning_rate": 4.886666666666667e-07,
+      "loss": 0.002,
+      "reward": 1.7880208492279053,
+      "reward_std": 0.043543651700019836,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7880208492279053,
+      "step": 767
+    },
+    {
+      "completion_length": 128.15625,
+      "epoch": 1.024,
+      "grad_norm": 3.9251324906622647,
+      "kl": 0.068359375,
+      "learning_rate": 4.879999999999999e-07,
+      "loss": 0.0027,
+      "reward": 1.7606863975524902,
+      "reward_std": 0.1641867309808731,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7919365167617798,
+      "step": 768
+    },
+    {
+      "completion_length": 122.6875,
+      "epoch": 1.0253333333333334,
+      "grad_norm": 1.791667118589574,
+      "kl": 0.041015625,
+      "learning_rate": 4.873333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.9010417461395264,
+      "reward_std": 0.06733439117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9010416865348816,
+      "step": 769
+    },
+    {
+      "completion_length": 119.96875,
+      "epoch": 1.0266666666666666,
+      "grad_norm": 1.2277827307559537,
+      "kl": 0.028564453125,
+      "learning_rate": 4.866666666666666e-07,
+      "loss": 0.0011,
+      "reward": 1.9067708253860474,
+      "reward_std": 0.007864408195018768,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9067708253860474,
+      "step": 770
+    },
+    {
+      "completion_length": 125.75,
+      "epoch": 1.028,
+      "grad_norm": 0.7119045009384157,
+      "kl": 0.03076171875,
+      "learning_rate": 4.86e-07,
+      "loss": 0.0012,
+      "reward": 1.876562476158142,
+      "reward_std": 0.07529377937316895,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.9078124761581421,
+      "step": 771
+    },
+    {
+      "completion_length": 130.6875,
+      "epoch": 1.0293333333333334,
+      "grad_norm": 1.7293849130049546,
+      "kl": 0.08251953125,
+      "learning_rate": 4.853333333333333e-07,
+      "loss": 0.0033,
+      "reward": 1.8372396230697632,
+      "reward_std": 0.17288917303085327,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8528645634651184,
+      "step": 772
+    },
+    {
+      "completion_length": 122.390625,
+      "epoch": 1.0306666666666666,
+      "grad_norm": 3.0062359075972305,
+      "kl": 0.043701171875,
+      "learning_rate": 4.846666666666667e-07,
+      "loss": 0.0017,
+      "reward": 1.7859375476837158,
+      "reward_std": 0.04479166492819786,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.785937488079071,
+      "step": 773
+    },
+    {
+      "completion_length": 123.90625,
+      "epoch": 1.032,
+      "grad_norm": 3.587740477438114,
+      "kl": 0.0263671875,
+      "learning_rate": 4.839999999999999e-07,
+      "loss": 0.0011,
+      "reward": 1.8932292461395264,
+      "reward_std": 0.15410654246807098,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9088541865348816,
+      "step": 774
+    },
+    {
+      "completion_length": 125.671875,
+      "epoch": 1.0333333333333334,
+      "grad_norm": 1.6500776458626023,
+      "kl": 0.05029296875,
+      "learning_rate": 4.833333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.853124976158142,
+      "reward_std": 0.15316304564476013,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8999999761581421,
+      "step": 775
+    },
+    {
+      "completion_length": 129.34375,
+      "epoch": 1.0346666666666666,
+      "grad_norm": 1.0993435239451366,
+      "kl": 0.060546875,
+      "learning_rate": 4.826666666666666e-07,
+      "loss": 0.0024,
+      "reward": 1.8742187023162842,
+      "reward_std": 0.0703125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.874218761920929,
+      "step": 776
+    },
+    {
+      "completion_length": 129.0625,
+      "epoch": 1.036,
+      "grad_norm": 1.4416224039693892,
+      "kl": 0.04052734375,
+      "learning_rate": 4.82e-07,
+      "loss": 0.0016,
+      "reward": 1.8689360618591309,
+      "reward_std": 0.11583594977855682,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8845610618591309,
+      "step": 777
+    },
+    {
+      "completion_length": 123.796875,
+      "epoch": 1.0373333333333334,
+      "grad_norm": 1.0417545254264013,
+      "kl": 0.048095703125,
+      "learning_rate": 4.813333333333334e-07,
+      "loss": 0.0019,
+      "reward": 1.965104103088379,
+      "reward_std": 0.0278890673071146,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9651041030883789,
+      "step": 778
+    },
+    {
+      "completion_length": 122.421875,
+      "epoch": 1.0386666666666666,
+      "grad_norm": 1.7436358423374174,
+      "kl": 0.039306640625,
+      "learning_rate": 4.806666666666667e-07,
+      "loss": 0.0016,
+      "reward": 1.8567708730697632,
+      "reward_std": 0.14640678465366364,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8723958134651184,
+      "step": 779
+    },
+    {
+      "completion_length": 134.484375,
+      "epoch": 1.04,
+      "grad_norm": 1.4359458018362115,
+      "kl": 0.039794921875,
+      "learning_rate": 4.8e-07,
+      "loss": 0.0016,
+      "reward": 1.8203125,
+      "reward_std": 0.060514599084854126,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8203125596046448,
+      "step": 780
+    },
+    {
+      "completion_length": 113.890625,
+      "epoch": 1.0413333333333332,
+      "grad_norm": 0.8957785555756188,
+      "kl": 0.0223388671875,
+      "learning_rate": 4.793333333333333e-07,
+      "loss": 0.0009,
+      "reward": 1.845312476158142,
+      "reward_std": 0.04161445423960686,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8453124761581421,
+      "step": 781
+    },
+    {
+      "completion_length": 119.0,
+      "epoch": 1.0426666666666666,
+      "grad_norm": 2.2263754208628885,
+      "kl": 0.06884765625,
+      "learning_rate": 4.786666666666667e-07,
+      "loss": 0.0027,
+      "reward": 1.8718750476837158,
+      "reward_std": 0.04119478911161423,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.871874988079071,
+      "step": 782
+    },
+    {
+      "completion_length": 124.53125,
+      "epoch": 1.044,
+      "grad_norm": 1.4323449128696963,
+      "kl": 0.05029296875,
+      "learning_rate": 4.779999999999999e-07,
+      "loss": 0.002,
+      "reward": 1.8599331378936768,
+      "reward_std": 0.036066845059394836,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8599330186843872,
+      "step": 783
+    },
+    {
+      "completion_length": 116.890625,
+      "epoch": 1.0453333333333332,
+      "grad_norm": 1.9021783646459813,
+      "kl": 0.041748046875,
+      "learning_rate": 4.773333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.816927194595337,
+      "reward_std": 0.08093348890542984,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8169270753860474,
+      "step": 784
+    },
+    {
+      "completion_length": 123.84375,
+      "epoch": 1.0466666666666666,
+      "grad_norm": 1.7498577144275584,
+      "kl": 0.0654296875,
+      "learning_rate": 4.7666666666666667e-07,
+      "loss": 0.0026,
+      "reward": 1.89453125,
+      "reward_std": 0.0698690265417099,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.89453125,
+      "step": 785
+    },
+    {
+      "completion_length": 103.96875,
+      "epoch": 1.048,
+      "grad_norm": 0.7695817970449104,
+      "kl": 0.0225830078125,
+      "learning_rate": 4.76e-07,
+      "loss": 0.0009,
+      "reward": 1.859375,
+      "reward_std": 0.03125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.859375,
+      "step": 786
+    },
+    {
+      "completion_length": 129.234375,
+      "epoch": 1.0493333333333332,
+      "grad_norm": 2.261684961322933,
+      "kl": 0.05029296875,
+      "learning_rate": 4.7533333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.6514136791229248,
+      "reward_std": 0.15336871147155762,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.6826636791229248,
+      "step": 787
+    },
+    {
+      "completion_length": 127.15625,
+      "epoch": 1.0506666666666666,
+      "grad_norm": 2.6334863201786907,
+      "kl": 0.057861328125,
+      "learning_rate": 4.746666666666667e-07,
+      "loss": 0.0023,
+      "reward": 1.8526041507720947,
+      "reward_std": 0.07494266331195831,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8526041507720947,
+      "step": 788
+    },
+    {
+      "completion_length": 127.1875,
+      "epoch": 1.052,
+      "grad_norm": 2.0412478941326526,
+      "kl": 0.05126953125,
+      "learning_rate": 4.7399999999999993e-07,
+      "loss": 0.0021,
+      "reward": 1.6812500953674316,
+      "reward_std": 0.19145266711711884,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6968749761581421,
+      "step": 789
+    },
+    {
+      "completion_length": 116.390625,
+      "epoch": 1.0533333333333332,
+      "grad_norm": 1.2226769339876669,
+      "kl": 0.04443359375,
+      "learning_rate": 4.733333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.9375,
+      "reward_std": 0.0624999925494194,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9375,
+      "step": 790
+    },
+    {
+      "completion_length": 123.265625,
+      "epoch": 1.0546666666666666,
+      "grad_norm": 2.7173023489686456,
+      "kl": 0.07421875,
+      "learning_rate": 4.7266666666666664e-07,
+      "loss": 0.003,
+      "reward": 1.703125,
+      "reward_std": 0.19336450099945068,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.703125,
+      "step": 791
+    },
+    {
+      "completion_length": 110.03125,
+      "epoch": 1.056,
+      "grad_norm": 4.578656214841765,
+      "kl": 0.06787109375,
+      "learning_rate": 4.7199999999999994e-07,
+      "loss": 0.0027,
+      "reward": 1.8270833492279053,
+      "reward_std": 0.03749999776482582,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8270833492279053,
+      "step": 792
+    },
+    {
+      "completion_length": 118.671875,
+      "epoch": 1.0573333333333332,
+      "grad_norm": 1.7264046559720578,
+      "kl": 0.06689453125,
+      "learning_rate": 4.713333333333333e-07,
+      "loss": 0.0027,
+      "reward": 1.7817708253860474,
+      "reward_std": 0.0629202201962471,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7817708849906921,
+      "step": 793
+    },
+    {
+      "completion_length": 115.640625,
+      "epoch": 1.0586666666666666,
+      "grad_norm": 1.1831478411120104,
+      "kl": 0.050048828125,
+      "learning_rate": 4.7066666666666665e-07,
+      "loss": 0.002,
+      "reward": 1.8307292461395264,
+      "reward_std": 0.0554041862487793,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8307291865348816,
+      "step": 794
+    },
+    {
+      "completion_length": 125.34375,
+      "epoch": 1.06,
+      "grad_norm": 4.534277589371235,
+      "kl": 0.039794921875,
+      "learning_rate": 4.6999999999999995e-07,
+      "loss": 0.0016,
+      "reward": 1.80078125,
+      "reward_std": 0.17988571524620056,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.81640625,
+      "step": 795
+    },
+    {
+      "completion_length": 117.234375,
+      "epoch": 1.0613333333333332,
+      "grad_norm": 0.8600838684793085,
+      "kl": 0.031982421875,
+      "learning_rate": 4.693333333333333e-07,
+      "loss": 0.0013,
+      "reward": 1.859375,
+      "reward_std": 0.12717358767986298,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.890625,
+      "step": 796
+    },
+    {
+      "completion_length": 119.65625,
+      "epoch": 1.0626666666666666,
+      "grad_norm": 1.653071907663965,
+      "kl": 0.05126953125,
+      "learning_rate": 4.6866666666666665e-07,
+      "loss": 0.002,
+      "reward": 1.65625,
+      "reward_std": 0.13466878235340118,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.703125,
+      "step": 797
+    },
+    {
+      "completion_length": 117.4375,
+      "epoch": 1.064,
+      "grad_norm": 1.3853664032329691,
+      "kl": 0.051513671875,
+      "learning_rate": 4.68e-07,
+      "loss": 0.0021,
+      "reward": 1.9330358505249023,
+      "reward_std": 0.047742415219545364,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9330357313156128,
+      "step": 798
+    },
+    {
+      "completion_length": 118.765625,
+      "epoch": 1.0653333333333332,
+      "grad_norm": 2.3318533915234627,
+      "kl": 0.050537109375,
+      "learning_rate": 4.673333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.828125,
+      "reward_std": 0.0520833358168602,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.828125,
+      "step": 799
+    },
+    {
+      "completion_length": 101.84375,
+      "epoch": 1.0666666666666667,
+      "grad_norm": 1.6464470187730653,
+      "kl": 0.046142578125,
+      "learning_rate": 4.6666666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.84765625,
+      "reward_std": 0.07335582375526428,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.84765625,
+      "step": 800
+    },
+    {
+      "completion_length": 115.890625,
+      "epoch": 1.068,
+      "grad_norm": 2.3443899248623947,
+      "kl": 0.0322265625,
+      "learning_rate": 4.66e-07,
+      "loss": 0.0013,
+      "reward": 1.8151042461395264,
+      "reward_std": 0.18741881847381592,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8307291269302368,
+      "step": 801
+    },
+    {
+      "completion_length": 118.53125,
+      "epoch": 1.0693333333333332,
+      "grad_norm": 6.019118281338294,
+      "kl": 0.0498046875,
+      "learning_rate": 4.653333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.8120163679122925,
+      "reward_std": 0.06389638036489487,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8120163679122925,
+      "step": 802
+    },
+    {
+      "completion_length": 108.375,
+      "epoch": 1.0706666666666667,
+      "grad_norm": 1.6788757837129933,
+      "kl": 0.048095703125,
+      "learning_rate": 4.6466666666666667e-07,
+      "loss": 0.0019,
+      "reward": 1.8203125,
+      "reward_std": 0.10056979209184647,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8203125,
+      "step": 803
+    },
+    {
+      "completion_length": 111.484375,
+      "epoch": 1.072,
+      "grad_norm": 2.2710345793232607,
+      "kl": 0.0673828125,
+      "learning_rate": 4.64e-07,
+      "loss": 0.0027,
+      "reward": 1.9026042222976685,
+      "reward_std": 0.06450854241847992,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9026042222976685,
+      "step": 804
+    },
+    {
+      "completion_length": 131.59375,
+      "epoch": 1.0733333333333333,
+      "grad_norm": 2.050415321674539,
+      "kl": 0.061767578125,
+      "learning_rate": 4.633333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.7486979961395264,
+      "reward_std": 0.1229378879070282,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7643229365348816,
+      "step": 805
+    },
+    {
+      "completion_length": 128.59375,
+      "epoch": 1.0746666666666667,
+      "grad_norm": 1.8208005842165709,
+      "kl": 0.07568359375,
+      "learning_rate": 4.6266666666666663e-07,
+      "loss": 0.003,
+      "reward": 1.7679688930511475,
+      "reward_std": 0.06401349604129791,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7679687738418579,
+      "step": 806
+    },
+    {
+      "completion_length": 115.90625,
+      "epoch": 1.076,
+      "grad_norm": 1.0353498347896888,
+      "kl": 0.0478515625,
+      "learning_rate": 4.62e-07,
+      "loss": 0.0019,
+      "reward": 1.8338541984558105,
+      "reward_std": 0.07254272699356079,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8338541388511658,
+      "step": 807
+    },
+    {
+      "completion_length": 118.1875,
+      "epoch": 1.0773333333333333,
+      "grad_norm": 1.1039341584457336,
+      "kl": 0.051025390625,
+      "learning_rate": 4.613333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.7918062210083008,
+      "reward_std": 0.12501926720142365,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.807431161403656,
+      "step": 808
+    },
+    {
+      "completion_length": 118.625,
+      "epoch": 1.0786666666666667,
+      "grad_norm": 1.5039886155127835,
+      "kl": 0.05322265625,
+      "learning_rate": 4.6066666666666664e-07,
+      "loss": 0.0021,
+      "reward": 1.8322917222976685,
+      "reward_std": 0.09581640362739563,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8322917222976685,
+      "step": 809
+    },
+    {
+      "completion_length": 132.078125,
+      "epoch": 1.08,
+      "grad_norm": 2.228691166798577,
+      "kl": 0.04150390625,
+      "learning_rate": 4.6e-07,
+      "loss": 0.0017,
+      "reward": 1.797395944595337,
+      "reward_std": 0.12674032151699066,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8130208253860474,
+      "step": 810
+    },
+    {
+      "completion_length": 139.96875,
+      "epoch": 1.0813333333333333,
+      "grad_norm": 2.9218658987812214,
+      "kl": 0.05322265625,
+      "learning_rate": 4.593333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.745833396911621,
+      "reward_std": 0.16111209988594055,
+      "rewards/format_reward": 0.90625,
+      "rewards/iou_reward": 0.8395833969116211,
+      "step": 811
+    },
+    {
+      "completion_length": 124.125,
+      "epoch": 1.0826666666666667,
+      "grad_norm": 6.913901380413493,
+      "kl": 0.08349609375,
+      "learning_rate": 4.5866666666666664e-07,
+      "loss": 0.0033,
+      "reward": 1.7664062976837158,
+      "reward_std": 0.05949850380420685,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.766406238079071,
+      "step": 812
+    },
+    {
+      "completion_length": 122.65625,
+      "epoch": 1.084,
+      "grad_norm": 1.7730427550231211,
+      "kl": 0.04833984375,
+      "learning_rate": 4.58e-07,
+      "loss": 0.0019,
+      "reward": 1.8494791984558105,
+      "reward_std": 0.09096317738294601,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8651041388511658,
+      "step": 813
+    },
+    {
+      "completion_length": 116.9375,
+      "epoch": 1.0853333333333333,
+      "grad_norm": 4.6439583625400225,
+      "kl": 0.053955078125,
+      "learning_rate": 4.573333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.8880208730697632,
+      "reward_std": 0.05831329524517059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8880208730697632,
+      "step": 814
+    },
+    {
+      "completion_length": 122.578125,
+      "epoch": 1.0866666666666667,
+      "grad_norm": 1.4563116703615693,
+      "kl": 0.046142578125,
+      "learning_rate": 4.5666666666666665e-07,
+      "loss": 0.0018,
+      "reward": 1.839583396911621,
+      "reward_std": 0.055815689265728,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8395833969116211,
+      "step": 815
+    },
+    {
+      "completion_length": 117.453125,
+      "epoch": 1.088,
+      "grad_norm": 10.961115539397138,
+      "kl": 0.0556640625,
+      "learning_rate": 4.56e-07,
+      "loss": 0.0022,
+      "reward": 1.8523437976837158,
+      "reward_std": 0.061213988810777664,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.852343738079071,
+      "step": 816
+    },
+    {
+      "completion_length": 121.265625,
+      "epoch": 1.0893333333333333,
+      "grad_norm": 1.032781330971178,
+      "kl": 0.0419921875,
+      "learning_rate": 4.553333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.8072917461395264,
+      "reward_std": 0.13950318098068237,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8385416865348816,
+      "step": 817
+    },
+    {
+      "completion_length": 122.453125,
+      "epoch": 1.0906666666666667,
+      "grad_norm": 1.639647441041039,
+      "kl": 0.058837890625,
+      "learning_rate": 4.5466666666666666e-07,
+      "loss": 0.0024,
+      "reward": 1.8216146230697632,
+      "reward_std": 0.0689169242978096,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8216145634651184,
+      "step": 818
+    },
+    {
+      "completion_length": 110.859375,
+      "epoch": 1.092,
+      "grad_norm": 1.9691265673220915,
+      "kl": 0.0703125,
+      "learning_rate": 4.54e-07,
+      "loss": 0.0028,
+      "reward": 1.8072917461395264,
+      "reward_std": 0.08994559198617935,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8072916865348816,
+      "step": 819
+    },
+    {
+      "completion_length": 124.90625,
+      "epoch": 1.0933333333333333,
+      "grad_norm": 1.15180492018641,
+      "kl": 0.04833984375,
+      "learning_rate": 4.5333333333333326e-07,
+      "loss": 0.0019,
+      "reward": 1.8515625,
+      "reward_std": 0.08295939117670059,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8671875,
+      "step": 820
+    },
+    {
+      "completion_length": 118.15625,
+      "epoch": 1.0946666666666667,
+      "grad_norm": 1.4644742153419448,
+      "kl": 0.06640625,
+      "learning_rate": 4.526666666666666e-07,
+      "loss": 0.0026,
+      "reward": 1.8114583492279053,
+      "reward_std": 0.047680728137493134,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8114583492279053,
+      "step": 821
+    },
+    {
+      "completion_length": 115.734375,
+      "epoch": 1.096,
+      "grad_norm": 2.0763307648463347,
+      "kl": 0.06591796875,
+      "learning_rate": 4.5199999999999997e-07,
+      "loss": 0.0026,
+      "reward": 1.7044271230697632,
+      "reward_std": 0.07578805834054947,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7044271230697632,
+      "step": 822
+    },
+    {
+      "completion_length": 121.453125,
+      "epoch": 1.0973333333333333,
+      "grad_norm": 1.1417296304119358,
+      "kl": 0.039306640625,
+      "learning_rate": 4.5133333333333327e-07,
+      "loss": 0.0016,
+      "reward": 1.8354166746139526,
+      "reward_std": 0.1355636715888977,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8666666746139526,
+      "step": 823
+    },
+    {
+      "completion_length": 119.03125,
+      "epoch": 1.0986666666666667,
+      "grad_norm": 3.9833652073335455,
+      "kl": 0.0286865234375,
+      "learning_rate": 4.506666666666666e-07,
+      "loss": 0.0011,
+      "reward": 1.9361979961395264,
+      "reward_std": 0.08035522699356079,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9361979365348816,
+      "step": 824
+    },
+    {
+      "completion_length": 120.140625,
+      "epoch": 1.1,
+      "grad_norm": 1.1577534200297572,
+      "kl": 0.0322265625,
+      "learning_rate": 4.5e-07,
+      "loss": 0.0013,
+      "reward": 1.8666666746139526,
+      "reward_std": 0.08237498998641968,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8666666746139526,
+      "step": 825
+    },
+    {
+      "completion_length": 123.75,
+      "epoch": 1.1013333333333333,
+      "grad_norm": 2.4450268445730514,
+      "kl": 0.052001953125,
+      "learning_rate": 4.493333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.67578125,
+      "reward_std": 0.12385299801826477,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.70703125,
+      "step": 826
+    },
+    {
+      "completion_length": 125.859375,
+      "epoch": 1.1026666666666667,
+      "grad_norm": 2.8320806948835493,
+      "kl": 0.053466796875,
+      "learning_rate": 4.4866666666666663e-07,
+      "loss": 0.0021,
+      "reward": 1.8046875,
+      "reward_std": 0.06373751908540726,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8046875,
+      "step": 827
+    },
+    {
+      "completion_length": 117.234375,
+      "epoch": 1.104,
+      "grad_norm": 2.3867878271510277,
+      "kl": 0.06640625,
+      "learning_rate": 4.48e-07,
+      "loss": 0.0027,
+      "reward": 1.8843750953674316,
+      "reward_std": 0.05944902449846268,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8843750357627869,
+      "step": 828
+    },
+    {
+      "completion_length": 121.6875,
+      "epoch": 1.1053333333333333,
+      "grad_norm": 1.1633211221170292,
+      "kl": 0.0556640625,
+      "learning_rate": 4.4733333333333334e-07,
+      "loss": 0.0022,
+      "reward": 1.9348958730697632,
+      "reward_std": 0.015389068983495235,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9348958730697632,
+      "step": 829
+    },
+    {
+      "completion_length": 127.921875,
+      "epoch": 1.1066666666666667,
+      "grad_norm": 1.9598481012991882,
+      "kl": 0.05126953125,
+      "learning_rate": 4.4666666666666664e-07,
+      "loss": 0.002,
+      "reward": 1.7403645515441895,
+      "reward_std": 0.12087349593639374,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7403646111488342,
+      "step": 830
+    },
+    {
+      "completion_length": 119.984375,
+      "epoch": 1.108,
+      "grad_norm": 1.558710354990513,
+      "kl": 0.04443359375,
+      "learning_rate": 4.46e-07,
+      "loss": 0.0018,
+      "reward": 1.872656226158142,
+      "reward_std": 0.08139689266681671,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8726562261581421,
+      "step": 831
+    },
+    {
+      "completion_length": 125.546875,
+      "epoch": 1.1093333333333333,
+      "grad_norm": 2.299089124715246,
+      "kl": 0.037841796875,
+      "learning_rate": 4.4533333333333335e-07,
+      "loss": 0.0015,
+      "reward": 1.8515625,
+      "reward_std": 0.03527866303920746,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8515625596046448,
+      "step": 832
+    },
+    {
+      "completion_length": 112.5625,
+      "epoch": 1.1106666666666667,
+      "grad_norm": 2.4339471609798258,
+      "kl": 0.0615234375,
+      "learning_rate": 4.4466666666666665e-07,
+      "loss": 0.0025,
+      "reward": 1.9468750953674316,
+      "reward_std": 0.019682783633470535,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9468750357627869,
+      "step": 833
+    },
+    {
+      "completion_length": 116.25,
+      "epoch": 1.112,
+      "grad_norm": 2.0439504603016756,
+      "kl": 0.054443359375,
+      "learning_rate": 4.44e-07,
+      "loss": 0.0022,
+      "reward": 1.7989583015441895,
+      "reward_std": 0.08101406693458557,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7989583611488342,
+      "step": 834
+    },
+    {
+      "completion_length": 130.390625,
+      "epoch": 1.1133333333333333,
+      "grad_norm": 3.4450909475497027,
+      "kl": 0.0546875,
+      "learning_rate": 4.4333333333333336e-07,
+      "loss": 0.0022,
+      "reward": 1.7348090410232544,
+      "reward_std": 0.16536499559879303,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7660590410232544,
+      "step": 835
+    },
+    {
+      "completion_length": 114.90625,
+      "epoch": 1.1146666666666667,
+      "grad_norm": 1.9612466779626727,
+      "kl": 0.042236328125,
+      "learning_rate": 4.426666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.8721354007720947,
+      "reward_std": 0.1168135553598404,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8721354603767395,
+      "step": 836
+    },
+    {
+      "completion_length": 121.171875,
+      "epoch": 1.116,
+      "grad_norm": 1.2237056272300486,
+      "kl": 0.037109375,
+      "learning_rate": 4.4199999999999996e-07,
+      "loss": 0.0015,
+      "reward": 1.9166667461395264,
+      "reward_std": 0.0625,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9322916865348816,
+      "step": 837
+    },
+    {
+      "completion_length": 111.078125,
+      "epoch": 1.1173333333333333,
+      "grad_norm": 1.872645959138763,
+      "kl": 0.042236328125,
+      "learning_rate": 4.413333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.758072853088379,
+      "reward_std": 0.06604617089033127,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7580728530883789,
+      "step": 838
+    },
+    {
+      "completion_length": 131.546875,
+      "epoch": 1.1186666666666667,
+      "grad_norm": 1.414801271401935,
+      "kl": 0.046875,
+      "learning_rate": 4.406666666666666e-07,
+      "loss": 0.0019,
+      "reward": 1.877343773841858,
+      "reward_std": 0.13358840346336365,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8929687738418579,
+      "step": 839
+    },
+    {
+      "completion_length": 128.875,
+      "epoch": 1.12,
+      "grad_norm": 2.1627924465557506,
+      "kl": 0.033935546875,
+      "learning_rate": 4.3999999999999997e-07,
+      "loss": 0.0014,
+      "reward": 1.8984375,
+      "reward_std": 0.14687499403953552,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8984375,
+      "step": 840
+    },
+    {
+      "completion_length": 133.890625,
+      "epoch": 1.1213333333333333,
+      "grad_norm": 3.4189571782560555,
+      "kl": 0.0458984375,
+      "learning_rate": 4.393333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.8441964387893677,
+      "reward_std": 0.07347512245178223,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8441964387893677,
+      "step": 841
+    },
+    {
+      "completion_length": 123.5,
+      "epoch": 1.1226666666666667,
+      "grad_norm": 2.0816444919338037,
+      "kl": 0.045654296875,
+      "learning_rate": 4.386666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.6713541746139526,
+      "reward_std": 0.1549798846244812,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6869791746139526,
+      "step": 842
+    },
+    {
+      "completion_length": 125.34375,
+      "epoch": 1.124,
+      "grad_norm": 2.0854756673764525,
+      "kl": 0.045654296875,
+      "learning_rate": 4.38e-07,
+      "loss": 0.0018,
+      "reward": 1.866722583770752,
+      "reward_std": 0.07728107273578644,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8823475241661072,
+      "step": 843
+    },
+    {
+      "completion_length": 116.703125,
+      "epoch": 1.1253333333333333,
+      "grad_norm": 1.2087418933736995,
+      "kl": 0.041748046875,
+      "learning_rate": 4.3733333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.9041666984558105,
+      "reward_std": 0.06372594833374023,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.904166579246521,
+      "step": 844
+    },
+    {
+      "completion_length": 109.59375,
+      "epoch": 1.1266666666666667,
+      "grad_norm": 1.532611667463149,
+      "kl": 0.029296875,
+      "learning_rate": 4.3666666666666663e-07,
+      "loss": 0.0012,
+      "reward": 1.8883929252624512,
+      "reward_std": 0.10599623620510101,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8883929252624512,
+      "step": 845
+    },
+    {
+      "completion_length": 116.625,
+      "epoch": 1.1280000000000001,
+      "grad_norm": 1.930720012565364,
+      "kl": 0.0498046875,
+      "learning_rate": 4.36e-07,
+      "loss": 0.002,
+      "reward": 1.7687499523162842,
+      "reward_std": 0.08750000596046448,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.768750011920929,
+      "step": 846
+    },
+    {
+      "completion_length": 118.796875,
+      "epoch": 1.1293333333333333,
+      "grad_norm": 7.440824257699353,
+      "kl": 0.062255859375,
+      "learning_rate": 4.3533333333333334e-07,
+      "loss": 0.0025,
+      "reward": 1.8414063453674316,
+      "reward_std": 0.04003506898880005,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8414062261581421,
+      "step": 847
+    },
+    {
+      "completion_length": 116.59375,
+      "epoch": 1.1306666666666667,
+      "grad_norm": 7.833093400350314,
+      "kl": 0.049072265625,
+      "learning_rate": 4.3466666666666664e-07,
+      "loss": 0.002,
+      "reward": 1.918229103088379,
+      "reward_std": 0.09302215278148651,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9182291626930237,
+      "step": 848
+    },
+    {
+      "completion_length": 118.21875,
+      "epoch": 1.1320000000000001,
+      "grad_norm": 2.520717735675996,
+      "kl": 0.0419921875,
+      "learning_rate": 4.34e-07,
+      "loss": 0.0017,
+      "reward": 1.8229167461395264,
+      "reward_std": 0.12146097421646118,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8229166865348816,
+      "step": 849
+    },
+    {
+      "completion_length": 127.59375,
+      "epoch": 1.1333333333333333,
+      "grad_norm": 1.186797884011373,
+      "kl": 0.0615234375,
+      "learning_rate": 4.3333333333333335e-07,
+      "loss": 0.0025,
+      "reward": 1.9322917461395264,
+      "reward_std": 0.0520833283662796,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9322916865348816,
+      "step": 850
+    },
+    {
+      "completion_length": 125.6875,
+      "epoch": 1.1346666666666667,
+      "grad_norm": 2.0603417762871823,
+      "kl": 0.060791015625,
+      "learning_rate": 4.3266666666666665e-07,
+      "loss": 0.0024,
+      "reward": 1.7833333015441895,
+      "reward_std": 0.12182429432868958,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7989583015441895,
+      "step": 851
+    },
+    {
+      "completion_length": 122.5625,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 5.343236913807054,
+      "kl": 0.042724609375,
+      "learning_rate": 4.3199999999999995e-07,
+      "loss": 0.0017,
+      "reward": 1.796875,
+      "reward_std": 0.19259506464004517,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8281250596046448,
+      "step": 852
+    },
+    {
+      "completion_length": 123.296875,
+      "epoch": 1.1373333333333333,
+      "grad_norm": 2.893494621204026,
+      "kl": 0.032958984375,
+      "learning_rate": 4.313333333333333e-07,
+      "loss": 0.0013,
+      "reward": 1.9296875,
+      "reward_std": 0.05170939117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9296875,
+      "step": 853
+    },
+    {
+      "completion_length": 125.953125,
+      "epoch": 1.1386666666666667,
+      "grad_norm": 2.470754235272579,
+      "kl": 0.0625,
+      "learning_rate": 4.306666666666666e-07,
+      "loss": 0.0025,
+      "reward": 1.8458333015441895,
+      "reward_std": 0.06741437315940857,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8458333611488342,
+      "step": 854
+    },
+    {
+      "completion_length": 122.265625,
+      "epoch": 1.1400000000000001,
+      "grad_norm": 1.0853919137077617,
+      "kl": 0.051513671875,
+      "learning_rate": 4.2999999999999996e-07,
+      "loss": 0.0021,
+      "reward": 1.8312499523162842,
+      "reward_std": 0.05228766053915024,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8312499523162842,
+      "step": 855
+    },
+    {
+      "completion_length": 121.328125,
+      "epoch": 1.1413333333333333,
+      "grad_norm": 9.627829504639195,
+      "kl": 0.03955078125,
+      "learning_rate": 4.293333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.5885417461395264,
+      "reward_std": 0.160197451710701,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6041666865348816,
+      "step": 856
+    },
+    {
+      "completion_length": 119.203125,
+      "epoch": 1.1426666666666667,
+      "grad_norm": 1.0858669609914917,
+      "kl": 0.04541015625,
+      "learning_rate": 4.286666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.9471354484558105,
+      "reward_std": 0.017195381224155426,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9471354484558105,
+      "step": 857
+    },
+    {
+      "completion_length": 124.34375,
+      "epoch": 1.144,
+      "grad_norm": 1.5545638127099646,
+      "kl": 0.05615234375,
+      "learning_rate": 4.2799999999999997e-07,
+      "loss": 0.0022,
+      "reward": 1.9007067680358887,
+      "reward_std": 0.09648250043392181,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9163318872451782,
+      "step": 858
+    },
+    {
+      "completion_length": 112.1875,
+      "epoch": 1.1453333333333333,
+      "grad_norm": 1.3897158617396947,
+      "kl": 0.06005859375,
+      "learning_rate": 4.273333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.87109375,
+      "reward_std": 0.07370638847351074,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8710938096046448,
+      "step": 859
+    },
+    {
+      "completion_length": 126.9375,
+      "epoch": 1.1466666666666667,
+      "grad_norm": 0.9212438809297019,
+      "kl": 0.05126953125,
+      "learning_rate": 4.266666666666667e-07,
+      "loss": 0.002,
+      "reward": 1.9343750476837158,
+      "reward_std": 0.10625000298023224,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.9656250476837158,
+      "step": 860
+    },
+    {
+      "completion_length": 137.234375,
+      "epoch": 1.148,
+      "grad_norm": 1.7549455633276176,
+      "kl": 0.051513671875,
+      "learning_rate": 4.26e-07,
+      "loss": 0.0021,
+      "reward": 1.798437476158142,
+      "reward_std": 0.12812499701976776,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8296874761581421,
+      "step": 861
+    },
+    {
+      "completion_length": 125.390625,
+      "epoch": 1.1493333333333333,
+      "grad_norm": 1.8183075032670555,
+      "kl": 0.04736328125,
+      "learning_rate": 4.2533333333333333e-07,
+      "loss": 0.0019,
+      "reward": 1.7588541507720947,
+      "reward_std": 0.14481349289417267,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7588542103767395,
+      "step": 862
+    },
+    {
+      "completion_length": 121.34375,
+      "epoch": 1.1506666666666667,
+      "grad_norm": 1.0411480756275595,
+      "kl": 0.030517578125,
+      "learning_rate": 4.246666666666667e-07,
+      "loss": 0.0012,
+      "reward": 1.875,
+      "reward_std": 0.125,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.890625,
+      "step": 863
+    },
+    {
+      "completion_length": 127.46875,
+      "epoch": 1.152,
+      "grad_norm": 1.7609222251798204,
+      "kl": 0.051025390625,
+      "learning_rate": 4.24e-07,
+      "loss": 0.002,
+      "reward": 1.91015625,
+      "reward_std": 0.09619611501693726,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9101563096046448,
+      "step": 864
+    },
+    {
+      "completion_length": 124.015625,
+      "epoch": 1.1533333333333333,
+      "grad_norm": 0.5223366900209228,
+      "kl": 0.0390625,
+      "learning_rate": 4.2333333333333334e-07,
+      "loss": 0.0016,
+      "reward": 1.8104166984558105,
+      "reward_std": 0.0625,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8104166984558105,
+      "step": 865
+    },
+    {
+      "completion_length": 113.140625,
+      "epoch": 1.1546666666666667,
+      "grad_norm": 0.9053414081352652,
+      "kl": 0.03857421875,
+      "learning_rate": 4.226666666666667e-07,
+      "loss": 0.0015,
+      "reward": 1.9270833730697632,
+      "reward_std": 0.012028127908706665,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9270833730697632,
+      "step": 866
+    },
+    {
+      "completion_length": 129.671875,
+      "epoch": 1.156,
+      "grad_norm": 10.320589772494074,
+      "kl": 0.07080078125,
+      "learning_rate": 4.2199999999999994e-07,
+      "loss": 0.0028,
+      "reward": 1.672536015510559,
+      "reward_std": 0.10358183830976486,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7037861347198486,
+      "step": 867
+    },
+    {
+      "completion_length": 119.59375,
+      "epoch": 1.1573333333333333,
+      "grad_norm": 1.3731225507698772,
+      "kl": 0.048095703125,
+      "learning_rate": 4.213333333333333e-07,
+      "loss": 0.0019,
+      "reward": 1.8723958730697632,
+      "reward_std": 0.0677083283662796,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8723958730697632,
+      "step": 868
+    },
+    {
+      "completion_length": 115.8125,
+      "epoch": 1.1586666666666667,
+      "grad_norm": 0.11781918938180737,
+      "kl": 0.04150390625,
+      "learning_rate": 4.2066666666666665e-07,
+      "loss": 0.0017,
+      "reward": 1.9375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9375,
+      "step": 869
+    },
+    {
+      "completion_length": 127.90625,
+      "epoch": 1.16,
+      "grad_norm": 2.103799416563806,
+      "kl": 0.078125,
+      "learning_rate": 4.1999999999999995e-07,
+      "loss": 0.0031,
+      "reward": 1.8875000476837158,
+      "reward_std": 0.07566770911216736,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8875000476837158,
+      "step": 870
+    },
+    {
+      "completion_length": 111.640625,
+      "epoch": 1.1613333333333333,
+      "grad_norm": 2.060207306295549,
+      "kl": 0.04443359375,
+      "learning_rate": 4.193333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.8359375,
+      "reward_std": 0.09575854241847992,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8359375,
+      "step": 871
+    },
+    {
+      "completion_length": 115.015625,
+      "epoch": 1.1626666666666667,
+      "grad_norm": 14.973459354292864,
+      "kl": 0.0615234375,
+      "learning_rate": 4.1866666666666666e-07,
+      "loss": 0.0025,
+      "reward": 1.8065104484558105,
+      "reward_std": 0.12804752588272095,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8065104484558105,
+      "step": 872
+    },
+    {
+      "completion_length": 120.015625,
+      "epoch": 1.164,
+      "grad_norm": 2.6728511413529277,
+      "kl": 0.051025390625,
+      "learning_rate": 4.1799999999999996e-07,
+      "loss": 0.002,
+      "reward": 1.8348214626312256,
+      "reward_std": 0.1515042930841446,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.834821343421936,
+      "step": 873
+    },
+    {
+      "completion_length": 112.59375,
+      "epoch": 1.1653333333333333,
+      "grad_norm": 1.0622491239825587,
+      "kl": 0.041748046875,
+      "learning_rate": 4.173333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.9010417461395264,
+      "reward_std": 0.0729166641831398,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9166666865348816,
+      "step": 874
+    },
+    {
+      "completion_length": 113.15625,
+      "epoch": 1.1666666666666667,
+      "grad_norm": 1.7838625643138728,
+      "kl": 0.045654296875,
+      "learning_rate": 4.1666666666666667e-07,
+      "loss": 0.0018,
+      "reward": 1.80078125,
+      "reward_std": 0.0536092072725296,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.80078125,
+      "step": 875
+    },
+    {
+      "completion_length": 113.84375,
+      "epoch": 1.168,
+      "grad_norm": 3.6314633801129133,
+      "kl": 0.07373046875,
+      "learning_rate": 4.1599999999999997e-07,
+      "loss": 0.0029,
+      "reward": 1.8434152603149414,
+      "reward_std": 0.14774051308631897,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8434152603149414,
+      "step": 876
+    },
+    {
+      "completion_length": 116.5625,
+      "epoch": 1.1693333333333333,
+      "grad_norm": 1.5311429716967182,
+      "kl": 0.02783203125,
+      "learning_rate": 4.153333333333333e-07,
+      "loss": 0.0011,
+      "reward": 1.9010417461395264,
+      "reward_std": 0.12394770234823227,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.9322916865348816,
+      "step": 877
+    },
+    {
+      "completion_length": 130.046875,
+      "epoch": 1.1706666666666667,
+      "grad_norm": 1.4734333712411523,
+      "kl": 0.041748046875,
+      "learning_rate": 4.146666666666667e-07,
+      "loss": 0.0017,
+      "reward": 1.8046875,
+      "reward_std": 0.171875,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8359375,
+      "step": 878
+    },
+    {
+      "completion_length": 116.15625,
+      "epoch": 1.172,
+      "grad_norm": 3.083366204523948,
+      "kl": 0.048583984375,
+      "learning_rate": 4.14e-07,
+      "loss": 0.0019,
+      "reward": 1.9187500476837158,
+      "reward_std": 0.04908787086606026,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.918749988079071,
+      "step": 879
+    },
+    {
+      "completion_length": 120.828125,
+      "epoch": 1.1733333333333333,
+      "grad_norm": 1.442216867906842,
+      "kl": 0.05224609375,
+      "learning_rate": 4.1333333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.8515625,
+      "reward_std": 0.09313592314720154,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8828125,
+      "step": 880
+    },
+    {
+      "completion_length": 115.90625,
+      "epoch": 1.1746666666666667,
+      "grad_norm": 2.109725763409828,
+      "kl": 0.053955078125,
+      "learning_rate": 4.126666666666667e-07,
+      "loss": 0.0022,
+      "reward": 1.8406250476837158,
+      "reward_std": 0.10905580222606659,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.840624988079071,
+      "step": 881
+    },
+    {
+      "completion_length": 118.296875,
+      "epoch": 1.176,
+      "grad_norm": 1.6799325568687453,
+      "kl": 0.05517578125,
+      "learning_rate": 4.12e-07,
+      "loss": 0.0022,
+      "reward": 1.7330728769302368,
+      "reward_std": 0.07756409049034119,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7330729365348816,
+      "step": 882
+    },
+    {
+      "completion_length": 110.65625,
+      "epoch": 1.1773333333333333,
+      "grad_norm": 3.7757882927952005,
+      "kl": 0.04248046875,
+      "learning_rate": 4.113333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.6536458730697632,
+      "reward_std": 0.1664201021194458,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.6692708134651184,
+      "step": 883
+    },
+    {
+      "completion_length": 113.59375,
+      "epoch": 1.1786666666666668,
+      "grad_norm": 2.4721485739292146,
+      "kl": 0.042724609375,
+      "learning_rate": 4.1066666666666664e-07,
+      "loss": 0.0017,
+      "reward": 1.8528646230697632,
+      "reward_std": 0.0494791641831398,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8684896230697632,
+      "step": 884
+    },
+    {
+      "completion_length": 113.109375,
+      "epoch": 1.18,
+      "grad_norm": 2.01225591848778,
+      "kl": 0.037109375,
+      "learning_rate": 4.0999999999999994e-07,
+      "loss": 0.0015,
+      "reward": 1.8125,
+      "reward_std": 0.1041666641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8125,
+      "step": 885
+    },
+    {
+      "completion_length": 112.53125,
+      "epoch": 1.1813333333333333,
+      "grad_norm": 2.4637607934085026,
+      "kl": 0.07861328125,
+      "learning_rate": 4.093333333333333e-07,
+      "loss": 0.0031,
+      "reward": 1.7690104246139526,
+      "reward_std": 0.036834798753261566,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7690103650093079,
+      "step": 886
+    },
+    {
+      "completion_length": 115.765625,
+      "epoch": 1.1826666666666668,
+      "grad_norm": 2.971594739948031,
+      "kl": 0.06591796875,
+      "learning_rate": 4.0866666666666665e-07,
+      "loss": 0.0026,
+      "reward": 1.8523437976837158,
+      "reward_std": 0.14854075014591217,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8835936784744263,
+      "step": 887
+    },
+    {
+      "completion_length": 108.875,
+      "epoch": 1.184,
+      "grad_norm": 1.5254768268559622,
+      "kl": 0.072265625,
+      "learning_rate": 4.0799999999999995e-07,
+      "loss": 0.0029,
+      "reward": 1.8880208730697632,
+      "reward_std": 0.03879941999912262,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8880207538604736,
+      "step": 888
+    },
+    {
+      "completion_length": 117.1875,
+      "epoch": 1.1853333333333333,
+      "grad_norm": 2.6481289988250163,
+      "kl": 0.0673828125,
+      "learning_rate": 4.073333333333333e-07,
+      "loss": 0.0027,
+      "reward": 1.8221354484558105,
+      "reward_std": 0.09922562539577484,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8221354484558105,
+      "step": 889
+    },
+    {
+      "completion_length": 108.359375,
+      "epoch": 1.1866666666666668,
+      "grad_norm": 1.4714393377034276,
+      "kl": 0.0576171875,
+      "learning_rate": 4.0666666666666666e-07,
+      "loss": 0.0023,
+      "reward": 1.91015625,
+      "reward_std": 0.06186956167221069,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.91015625,
+      "step": 890
+    },
+    {
+      "completion_length": 123.953125,
+      "epoch": 1.188,
+      "grad_norm": 1.1313240237705484,
+      "kl": 0.0322265625,
+      "learning_rate": 4.06e-07,
+      "loss": 0.0013,
+      "reward": 1.90625,
+      "reward_std": 0.09858439117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.90625,
+      "step": 891
+    },
+    {
+      "completion_length": 113.703125,
+      "epoch": 1.1893333333333334,
+      "grad_norm": 4.827193559635821,
+      "kl": 0.07177734375,
+      "learning_rate": 4.053333333333333e-07,
+      "loss": 0.0029,
+      "reward": 1.8000000715255737,
+      "reward_std": 0.15464192628860474,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7999999523162842,
+      "step": 892
+    },
+    {
+      "completion_length": 109.046875,
+      "epoch": 1.1906666666666668,
+      "grad_norm": 1.155657452541192,
+      "kl": 0.041259765625,
+      "learning_rate": 4.0466666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.9166667461395264,
+      "reward_std": 0.045769236981868744,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9166666865348816,
+      "step": 893
+    },
+    {
+      "completion_length": 106.1875,
+      "epoch": 1.192,
+      "grad_norm": 1.1937967043525715,
+      "kl": 0.04052734375,
+      "learning_rate": 4.04e-07,
+      "loss": 0.0016,
+      "reward": 1.8828125,
+      "reward_std": 0.078125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8828125,
+      "step": 894
+    },
+    {
+      "completion_length": 113.078125,
+      "epoch": 1.1933333333333334,
+      "grad_norm": 1.5571264821475699,
+      "kl": 0.048583984375,
+      "learning_rate": 4.033333333333333e-07,
+      "loss": 0.0019,
+      "reward": 1.7372395992279053,
+      "reward_std": 0.10239964723587036,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7372395992279053,
+      "step": 895
+    },
+    {
+      "completion_length": 107.71875,
+      "epoch": 1.1946666666666665,
+      "grad_norm": 1.7090925297375608,
+      "kl": 0.048095703125,
+      "learning_rate": 4.0266666666666667e-07,
+      "loss": 0.0019,
+      "reward": 1.8111978769302368,
+      "reward_std": 0.04306696355342865,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8268229365348816,
+      "step": 896
+    },
+    {
+      "completion_length": 117.453125,
+      "epoch": 1.196,
+      "grad_norm": 1.95138814641999,
+      "kl": 0.038818359375,
+      "learning_rate": 4.02e-07,
+      "loss": 0.0016,
+      "reward": 1.8283854722976685,
+      "reward_std": 0.0986533910036087,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8440104126930237,
+      "step": 897
+    },
+    {
+      "completion_length": 103.40625,
+      "epoch": 1.1973333333333334,
+      "grad_norm": 1.4556952773830438,
+      "kl": 0.05712890625,
+      "learning_rate": 4.0133333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.9010417461395264,
+      "reward_std": 0.07452812790870667,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9010416269302368,
+      "step": 898
+    },
+    {
+      "completion_length": 118.28125,
+      "epoch": 1.1986666666666665,
+      "grad_norm": 1.660534472270248,
+      "kl": 0.040283203125,
+      "learning_rate": 4.0066666666666663e-07,
+      "loss": 0.0016,
+      "reward": 1.7489583492279053,
+      "reward_std": 0.08750000596046448,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7489583492279053,
+      "step": 899
+    },
+    {
+      "completion_length": 113.34375,
+      "epoch": 1.2,
+      "grad_norm": 2.2503539459391444,
+      "kl": 0.060546875,
+      "learning_rate": 4e-07,
+      "loss": 0.0024,
+      "reward": 1.7768229246139526,
+      "reward_std": 0.20604297518730164,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7924479246139526,
+      "step": 900
+    },
+    {
+      "completion_length": 119.453125,
+      "epoch": 1.2013333333333334,
+      "grad_norm": 1.8449436014854337,
+      "kl": 0.04638671875,
+      "learning_rate": 3.993333333333333e-07,
+      "loss": 0.0019,
+      "reward": 1.920312523841858,
+      "reward_std": 0.08265465497970581,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9359375238418579,
+      "step": 901
+    },
+    {
+      "completion_length": 123.703125,
+      "epoch": 1.2026666666666666,
+      "grad_norm": 2.1836556483489993,
+      "kl": 0.0771484375,
+      "learning_rate": 3.9866666666666664e-07,
+      "loss": 0.0031,
+      "reward": 1.776822805404663,
+      "reward_std": 0.14915332198143005,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7924479246139526,
+      "step": 902
+    },
+    {
+      "completion_length": 134.546875,
+      "epoch": 1.204,
+      "grad_norm": 4.977291709915505,
+      "kl": 0.054931640625,
+      "learning_rate": 3.98e-07,
+      "loss": 0.0022,
+      "reward": 1.69140625,
+      "reward_std": 0.22224073112010956,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.69140625,
+      "step": 903
+    },
+    {
+      "completion_length": 108.984375,
+      "epoch": 1.2053333333333334,
+      "grad_norm": 2.0620755734439347,
+      "kl": 0.06103515625,
+      "learning_rate": 3.973333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.8763021230697632,
+      "reward_std": 0.13917919993400574,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8919270634651184,
+      "step": 904
+    },
+    {
+      "completion_length": 117.265625,
+      "epoch": 1.2066666666666666,
+      "grad_norm": 2.092768736673733,
+      "kl": 0.05322265625,
+      "learning_rate": 3.9666666666666665e-07,
+      "loss": 0.0021,
+      "reward": 1.788802146911621,
+      "reward_std": 0.061276741325855255,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7888020873069763,
+      "step": 905
+    },
+    {
+      "completion_length": 116.171875,
+      "epoch": 1.208,
+      "grad_norm": 1.8447613556315272,
+      "kl": 0.05859375,
+      "learning_rate": 3.96e-07,
+      "loss": 0.0023,
+      "reward": 1.8052456378936768,
+      "reward_std": 0.05124864727258682,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8052455186843872,
+      "step": 906
+    },
+    {
+      "completion_length": 126.40625,
+      "epoch": 1.2093333333333334,
+      "grad_norm": 2.664804529817683,
+      "kl": 0.07568359375,
+      "learning_rate": 3.953333333333333e-07,
+      "loss": 0.003,
+      "reward": 1.8132812976837158,
+      "reward_std": 0.1959802210330963,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.844531238079071,
+      "step": 907
+    },
+    {
+      "completion_length": 116.390625,
+      "epoch": 1.2106666666666666,
+      "grad_norm": 1.7507342898967406,
+      "kl": 0.05224609375,
+      "learning_rate": 3.9466666666666665e-07,
+      "loss": 0.0021,
+      "reward": 1.982031226158142,
+      "reward_std": 0.015029380097985268,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9820312261581421,
+      "step": 908
+    },
+    {
+      "completion_length": 128.3125,
+      "epoch": 1.212,
+      "grad_norm": 28.944661777643425,
+      "kl": 0.07568359375,
+      "learning_rate": 3.94e-07,
+      "loss": 0.003,
+      "reward": 1.774999976158142,
+      "reward_std": 0.07198040187358856,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7906250357627869,
+      "step": 909
+    },
+    {
+      "completion_length": 118.25,
+      "epoch": 1.2133333333333334,
+      "grad_norm": 11.338900077378318,
+      "kl": 0.04931640625,
+      "learning_rate": 3.933333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.7408854961395264,
+      "reward_std": 0.07656250149011612,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7408854365348816,
+      "step": 910
+    },
+    {
+      "completion_length": 113.3125,
+      "epoch": 1.2146666666666666,
+      "grad_norm": 2.522045728247043,
+      "kl": 0.0556640625,
+      "learning_rate": 3.9266666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.8180804252624512,
+      "reward_std": 0.02664874494075775,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8180804252624512,
+      "step": 911
+    },
+    {
+      "completion_length": 120.546875,
+      "epoch": 1.216,
+      "grad_norm": 1.7352414877943985,
+      "kl": 0.054931640625,
+      "learning_rate": 3.92e-07,
+      "loss": 0.0022,
+      "reward": 1.7174479961395264,
+      "reward_std": 0.1569429337978363,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7486978769302368,
+      "step": 912
+    },
+    {
+      "completion_length": 117.46875,
+      "epoch": 1.2173333333333334,
+      "grad_norm": 1.4948046151391639,
+      "kl": 0.048583984375,
+      "learning_rate": 3.913333333333333e-07,
+      "loss": 0.0019,
+      "reward": 1.8611979484558105,
+      "reward_std": 0.0557345449924469,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8611979484558105,
+      "step": 913
+    },
+    {
+      "completion_length": 117.25,
+      "epoch": 1.2186666666666666,
+      "grad_norm": 1.6962887677524559,
+      "kl": 0.0625,
+      "learning_rate": 3.906666666666666e-07,
+      "loss": 0.0025,
+      "reward": 1.906622052192688,
+      "reward_std": 0.09537667036056519,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.906622052192688,
+      "step": 914
+    },
+    {
+      "completion_length": 120.0625,
+      "epoch": 1.22,
+      "grad_norm": 2.271650027377849,
+      "kl": 0.048583984375,
+      "learning_rate": 3.8999999999999997e-07,
+      "loss": 0.0019,
+      "reward": 1.9122395515441895,
+      "reward_std": 0.05052082985639572,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9122395515441895,
+      "step": 915
+    },
+    {
+      "completion_length": 110.421875,
+      "epoch": 1.2213333333333334,
+      "grad_norm": 2.128115117157102,
+      "kl": 0.064453125,
+      "learning_rate": 3.8933333333333327e-07,
+      "loss": 0.0026,
+      "reward": 1.902083396911621,
+      "reward_std": 0.11228654533624649,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9020833969116211,
+      "step": 916
+    },
+    {
+      "completion_length": 120.328125,
+      "epoch": 1.2226666666666666,
+      "grad_norm": 1.301351738857623,
+      "kl": 0.042724609375,
+      "learning_rate": 3.8866666666666663e-07,
+      "loss": 0.0017,
+      "reward": 1.953125,
+      "reward_std": 0.04650105535984039,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9531250596046448,
+      "step": 917
+    },
+    {
+      "completion_length": 118.5,
+      "epoch": 1.224,
+      "grad_norm": 0.6517622949100159,
+      "kl": 0.016845703125,
+      "learning_rate": 3.88e-07,
+      "loss": 0.0007,
+      "reward": 1.90625,
+      "reward_std": 0.03608439117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.90625,
+      "step": 918
+    },
+    {
+      "completion_length": 125.328125,
+      "epoch": 1.2253333333333334,
+      "grad_norm": 0.6370755403529003,
+      "kl": 0.03662109375,
+      "learning_rate": 3.873333333333333e-07,
+      "loss": 0.0015,
+      "reward": 1.761458396911621,
+      "reward_std": 0.03125,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.8239583373069763,
+      "step": 919
+    },
+    {
+      "completion_length": 118.65625,
+      "epoch": 1.2266666666666666,
+      "grad_norm": 1.7027463257511564,
+      "kl": 0.05029296875,
+      "learning_rate": 3.8666666666666664e-07,
+      "loss": 0.002,
+      "reward": 1.8424479961395264,
+      "reward_std": 0.1585344672203064,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8580729365348816,
+      "step": 920
+    },
+    {
+      "completion_length": 114.140625,
+      "epoch": 1.228,
+      "grad_norm": 1.1367981398783362,
+      "kl": 0.050048828125,
+      "learning_rate": 3.86e-07,
+      "loss": 0.002,
+      "reward": 1.9322917461395264,
+      "reward_std": 0.04650105535984039,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9322916269302368,
+      "step": 921
+    },
+    {
+      "completion_length": 107.515625,
+      "epoch": 1.2293333333333334,
+      "grad_norm": 1.2893711253119218,
+      "kl": 0.0439453125,
+      "learning_rate": 3.8533333333333334e-07,
+      "loss": 0.0018,
+      "reward": 1.8953125476837158,
+      "reward_std": 0.02487475797533989,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.895312488079071,
+      "step": 922
+    },
+    {
+      "completion_length": 123.203125,
+      "epoch": 1.2306666666666666,
+      "grad_norm": 4.186033389818498,
+      "kl": 0.07080078125,
+      "learning_rate": 3.8466666666666664e-07,
+      "loss": 0.0028,
+      "reward": 1.8541667461395264,
+      "reward_std": 0.17058861255645752,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8697916269302368,
+      "step": 923
+    },
+    {
+      "completion_length": 118.203125,
+      "epoch": 1.232,
+      "grad_norm": 1.6656958840678298,
+      "kl": 0.054443359375,
+      "learning_rate": 3.84e-07,
+      "loss": 0.0022,
+      "reward": 1.896093726158142,
+      "reward_std": 0.14728695154190063,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8960937857627869,
+      "step": 924
+    },
+    {
+      "completion_length": 116.125,
+      "epoch": 1.2333333333333334,
+      "grad_norm": 1.0624643076917644,
+      "kl": 0.054931640625,
+      "learning_rate": 3.8333333333333335e-07,
+      "loss": 0.0022,
+      "reward": 1.892187476158142,
+      "reward_std": 0.10894884169101715,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9078124761581421,
+      "step": 925
+    },
+    {
+      "completion_length": 111.265625,
+      "epoch": 1.2346666666666666,
+      "grad_norm": 1.0811302989556928,
+      "kl": 0.03662109375,
+      "learning_rate": 3.8266666666666665e-07,
+      "loss": 0.0015,
+      "reward": 1.8854167461395264,
+      "reward_std": 0.0729166641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8854166865348816,
+      "step": 926
+    },
+    {
+      "completion_length": 115.859375,
+      "epoch": 1.236,
+      "grad_norm": 4.82266909968235,
+      "kl": 0.0458984375,
+      "learning_rate": 3.82e-07,
+      "loss": 0.0018,
+      "reward": 1.7838542461395264,
+      "reward_std": 0.16507019102573395,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8307291269302368,
+      "step": 927
+    },
+    {
+      "completion_length": 125.34375,
+      "epoch": 1.2373333333333334,
+      "grad_norm": 1.7457567407944456,
+      "kl": 0.046142578125,
+      "learning_rate": 3.8133333333333336e-07,
+      "loss": 0.0018,
+      "reward": 1.8914062976837158,
+      "reward_std": 0.11759361624717712,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.907031238079071,
+      "step": 928
+    },
+    {
+      "completion_length": 117.796875,
+      "epoch": 1.2386666666666666,
+      "grad_norm": 1.4940755673733415,
+      "kl": 0.0390625,
+      "learning_rate": 3.8066666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.8385417461395264,
+      "reward_std": 0.1090010553598404,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8385416865348816,
+      "step": 929
+    },
+    {
+      "completion_length": 116.28125,
+      "epoch": 1.24,
+      "grad_norm": 1.5898409188793303,
+      "kl": 0.041259765625,
+      "learning_rate": 3.7999999999999996e-07,
+      "loss": 0.0016,
+      "reward": 1.90625,
+      "reward_std": 0.1194177195429802,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9062500596046448,
+      "step": 930
+    },
+    {
+      "completion_length": 126.046875,
+      "epoch": 1.2413333333333334,
+      "grad_norm": 31.773034155997443,
+      "kl": 0.039306640625,
+      "learning_rate": 3.793333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.8699219226837158,
+      "reward_std": 0.024264685809612274,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8699219226837158,
+      "step": 931
+    },
+    {
+      "completion_length": 116.25,
+      "epoch": 1.2426666666666666,
+      "grad_norm": 3.6942980845325297,
+      "kl": 0.055908203125,
+      "learning_rate": 3.786666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.78125,
+      "reward_std": 0.14805246889591217,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7812499403953552,
+      "step": 932
+    },
+    {
+      "completion_length": 108.640625,
+      "epoch": 1.244,
+      "grad_norm": 1.7602486786375928,
+      "kl": 0.044677734375,
+      "learning_rate": 3.7799999999999997e-07,
+      "loss": 0.0018,
+      "reward": 1.875,
+      "reward_std": 0.10577812790870667,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.890625,
+      "step": 933
+    },
+    {
+      "completion_length": 120.203125,
+      "epoch": 1.2453333333333334,
+      "grad_norm": 1.3011577150281368,
+      "kl": 0.050048828125,
+      "learning_rate": 3.773333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.7997395992279053,
+      "reward_std": 0.05840543657541275,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.79973965883255,
+      "step": 934
+    },
+    {
+      "completion_length": 105.1875,
+      "epoch": 1.2466666666666666,
+      "grad_norm": 2.1009531283623613,
+      "kl": 0.052490234375,
+      "learning_rate": 3.766666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.8059896230697632,
+      "reward_std": 0.0755208283662796,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8216145634651184,
+      "step": 935
+    },
+    {
+      "completion_length": 116.234375,
+      "epoch": 1.248,
+      "grad_norm": 1.5204939323272653,
+      "kl": 0.036376953125,
+      "learning_rate": 3.76e-07,
+      "loss": 0.0015,
+      "reward": 1.8484375476837158,
+      "reward_std": 0.05048343911767006,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.848437488079071,
+      "step": 936
+    },
+    {
+      "completion_length": 126.1875,
+      "epoch": 1.2493333333333334,
+      "grad_norm": 2.050505410737942,
+      "kl": 0.0615234375,
+      "learning_rate": 3.7533333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.7856770753860474,
+      "reward_std": 0.03784627094864845,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7856771349906921,
+      "step": 937
+    },
+    {
+      "completion_length": 121.640625,
+      "epoch": 1.2506666666666666,
+      "grad_norm": 2.28325091069823,
+      "kl": 0.057373046875,
+      "learning_rate": 3.7466666666666663e-07,
+      "loss": 0.0023,
+      "reward": 1.8684896230697632,
+      "reward_std": 0.1015455573797226,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8684895038604736,
+      "step": 938
+    },
+    {
+      "completion_length": 105.765625,
+      "epoch": 1.252,
+      "grad_norm": 2.4875447504568147,
+      "kl": 0.060791015625,
+      "learning_rate": 3.74e-07,
+      "loss": 0.0024,
+      "reward": 1.8565104007720947,
+      "reward_std": 0.06153570115566254,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8565104007720947,
+      "step": 939
+    },
+    {
+      "completion_length": 115.8125,
+      "epoch": 1.2533333333333334,
+      "grad_norm": 1.3682247539661296,
+      "kl": 0.0380859375,
+      "learning_rate": 3.7333333333333334e-07,
+      "loss": 0.0015,
+      "reward": 1.796875,
+      "reward_std": 0.06315963715314865,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.796875,
+      "step": 940
+    },
+    {
+      "completion_length": 113.265625,
+      "epoch": 1.2546666666666666,
+      "grad_norm": 1.8743551164238246,
+      "kl": 0.0673828125,
+      "learning_rate": 3.7266666666666664e-07,
+      "loss": 0.0027,
+      "reward": 1.7804687023162842,
+      "reward_std": 0.1530085951089859,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7804688215255737,
+      "step": 941
+    },
+    {
+      "completion_length": 115.828125,
+      "epoch": 1.256,
+      "grad_norm": 1.3161954072318225,
+      "kl": 0.04833984375,
+      "learning_rate": 3.72e-07,
+      "loss": 0.0019,
+      "reward": 1.8844493627548218,
+      "reward_std": 0.05267859622836113,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8844493627548218,
+      "step": 942
+    },
+    {
+      "completion_length": 119.546875,
+      "epoch": 1.2573333333333334,
+      "grad_norm": 2.252781735639083,
+      "kl": 0.05322265625,
+      "learning_rate": 3.7133333333333335e-07,
+      "loss": 0.0021,
+      "reward": 1.9583333730697632,
+      "reward_std": 0.006014068145304918,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9583333730697632,
+      "step": 943
+    },
+    {
+      "completion_length": 120.390625,
+      "epoch": 1.2586666666666666,
+      "grad_norm": 1.4731370528485517,
+      "kl": 0.07568359375,
+      "learning_rate": 3.7066666666666665e-07,
+      "loss": 0.003,
+      "reward": 1.8606770038604736,
+      "reward_std": 0.060206107795238495,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8606771230697632,
+      "step": 944
+    },
+    {
+      "completion_length": 118.484375,
+      "epoch": 1.26,
+      "grad_norm": 1.4865309333469114,
+      "kl": 0.072265625,
+      "learning_rate": 3.7e-07,
+      "loss": 0.0029,
+      "reward": 1.7505208253860474,
+      "reward_std": 0.07198134809732437,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7661458849906921,
+      "step": 945
+    },
+    {
+      "completion_length": 119.09375,
+      "epoch": 1.2613333333333334,
+      "grad_norm": 1.537254392937319,
+      "kl": 0.0625,
+      "learning_rate": 3.693333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.7166666984558105,
+      "reward_std": 0.05712204799056053,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7166666984558105,
+      "step": 946
+    },
+    {
+      "completion_length": 119.296875,
+      "epoch": 1.2626666666666666,
+      "grad_norm": 2.563839809118098,
+      "kl": 0.052490234375,
+      "learning_rate": 3.686666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.915624976158142,
+      "reward_std": 0.10623905062675476,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9156249761581421,
+      "step": 947
+    },
+    {
+      "completion_length": 119.65625,
+      "epoch": 1.264,
+      "grad_norm": 1.7363473821593072,
+      "kl": 0.055419921875,
+      "learning_rate": 3.6799999999999996e-07,
+      "loss": 0.0022,
+      "reward": 1.9505208730697632,
+      "reward_std": 0.0885416641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9505208730697632,
+      "step": 948
+    },
+    {
+      "completion_length": 122.296875,
+      "epoch": 1.2653333333333334,
+      "grad_norm": 5.402874695227168,
+      "kl": 0.0517578125,
+      "learning_rate": 3.673333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.8255208730697632,
+      "reward_std": 0.14868229627609253,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8411458730697632,
+      "step": 949
+    },
+    {
+      "completion_length": 116.40625,
+      "epoch": 1.2666666666666666,
+      "grad_norm": 1.1836542264449919,
+      "kl": 0.06884765625,
+      "learning_rate": 3.666666666666666e-07,
+      "loss": 0.0028,
+      "reward": 1.7044271230697632,
+      "reward_std": 0.009895836934447289,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7044271230697632,
+      "step": 950
+    },
+    {
+      "completion_length": 126.234375,
+      "epoch": 1.268,
+      "grad_norm": 1.088605927152068,
+      "kl": 0.09765625,
+      "learning_rate": 3.6599999999999997e-07,
+      "loss": 0.0039,
+      "reward": 1.9375,
+      "reward_std": 0.078125,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.953125,
+      "step": 951
+    },
+    {
+      "completion_length": 127.703125,
+      "epoch": 1.2693333333333334,
+      "grad_norm": 35.756187129515844,
+      "kl": 0.053466796875,
+      "learning_rate": 3.653333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.6932291984558105,
+      "reward_std": 0.15512818098068237,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7244791984558105,
+      "step": 952
+    },
+    {
+      "completion_length": 122.75,
+      "epoch": 1.2706666666666666,
+      "grad_norm": 1.472452178359488,
+      "kl": 0.07177734375,
+      "learning_rate": 3.646666666666666e-07,
+      "loss": 0.0029,
+      "reward": 1.8638020753860474,
+      "reward_std": 0.0254686176776886,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8638021349906921,
+      "step": 953
+    },
+    {
+      "completion_length": 115.859375,
+      "epoch": 1.272,
+      "grad_norm": 1.7004874460585735,
+      "kl": 0.04736328125,
+      "learning_rate": 3.64e-07,
+      "loss": 0.0019,
+      "reward": 1.852083444595337,
+      "reward_std": 0.10437099635601044,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8520833253860474,
+      "step": 954
+    },
+    {
+      "completion_length": 130.671875,
+      "epoch": 1.2733333333333334,
+      "grad_norm": 1.45062934841549,
+      "kl": 0.04296875,
+      "learning_rate": 3.6333333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.8072917461395264,
+      "reward_std": 0.1241130605340004,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8072916865348816,
+      "step": 955
+    },
+    {
+      "completion_length": 124.9375,
+      "epoch": 1.2746666666666666,
+      "grad_norm": 2.239468198076027,
+      "kl": 0.06640625,
+      "learning_rate": 3.626666666666667e-07,
+      "loss": 0.0027,
+      "reward": 1.7829241752624512,
+      "reward_std": 0.023431366309523582,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7829241752624512,
+      "step": 956
+    },
+    {
+      "completion_length": 119.375,
+      "epoch": 1.276,
+      "grad_norm": 0.7356923440586032,
+      "kl": 0.050537109375,
+      "learning_rate": 3.62e-07,
+      "loss": 0.002,
+      "reward": 1.9739583730697632,
+      "reward_std": 0.043278127908706665,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9895833134651184,
+      "step": 957
+    },
+    {
+      "completion_length": 121.34375,
+      "epoch": 1.2773333333333334,
+      "grad_norm": 1.6290203317754914,
+      "kl": 0.04150390625,
+      "learning_rate": 3.6133333333333334e-07,
+      "loss": 0.0017,
+      "reward": 1.8302083015441895,
+      "reward_std": 0.1305021047592163,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.830208420753479,
+      "step": 958
+    },
+    {
+      "completion_length": 117.015625,
+      "epoch": 1.2786666666666666,
+      "grad_norm": 1.8064136185971251,
+      "kl": 0.053466796875,
+      "learning_rate": 3.606666666666667e-07,
+      "loss": 0.0021,
+      "reward": 1.9140625,
+      "reward_std": 0.0989583283662796,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9140625,
+      "step": 959
+    },
+    {
+      "completion_length": 128.78125,
+      "epoch": 1.28,
+      "grad_norm": 1.6822381662707175,
+      "kl": 0.052490234375,
+      "learning_rate": 3.6e-07,
+      "loss": 0.0021,
+      "reward": 1.7916667461395264,
+      "reward_std": 0.0677083283662796,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7916666865348816,
+      "step": 960
+    },
+    {
+      "completion_length": 131.21875,
+      "epoch": 1.2813333333333334,
+      "grad_norm": 1.8148403863588007,
+      "kl": 0.06103515625,
+      "learning_rate": 3.5933333333333335e-07,
+      "loss": 0.0024,
+      "reward": 1.8145833015441895,
+      "reward_std": 0.14608851075172424,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8302083611488342,
+      "step": 961
+    },
+    {
+      "completion_length": 119.609375,
+      "epoch": 1.2826666666666666,
+      "grad_norm": 2.6905133151556884,
+      "kl": 0.0751953125,
+      "learning_rate": 3.5866666666666665e-07,
+      "loss": 0.003,
+      "reward": 1.7997395992279053,
+      "reward_std": 0.16342607140541077,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7997395992279053,
+      "step": 962
+    },
+    {
+      "completion_length": 122.0,
+      "epoch": 1.284,
+      "grad_norm": 1.4714167142503947,
+      "kl": 0.04736328125,
+      "learning_rate": 3.5799999999999995e-07,
+      "loss": 0.0019,
+      "reward": 1.9309896230697632,
+      "reward_std": 0.05182025581598282,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9309895634651184,
+      "step": 963
+    },
+    {
+      "completion_length": 136.0625,
+      "epoch": 1.2853333333333334,
+      "grad_norm": 2.0845431019601794,
+      "kl": 0.05419921875,
+      "learning_rate": 3.573333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.7682292461395264,
+      "reward_std": 0.11299805343151093,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7838541269302368,
+      "step": 964
+    },
+    {
+      "completion_length": 133.140625,
+      "epoch": 1.2866666666666666,
+      "grad_norm": 1.3878586617261672,
+      "kl": 0.046875,
+      "learning_rate": 3.5666666666666666e-07,
+      "loss": 0.0019,
+      "reward": 1.7202754020690918,
+      "reward_std": 0.12279081344604492,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7515252828598022,
+      "step": 965
+    },
+    {
+      "completion_length": 124.46875,
+      "epoch": 1.288,
+      "grad_norm": 2.664287270564986,
+      "kl": 0.046875,
+      "learning_rate": 3.5599999999999996e-07,
+      "loss": 0.0019,
+      "reward": 1.840066909790039,
+      "reward_std": 0.09532460570335388,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8400669693946838,
+      "step": 966
+    },
+    {
+      "completion_length": 120.6875,
+      "epoch": 1.2893333333333334,
+      "grad_norm": 1.3807819174271905,
+      "kl": 0.031005859375,
+      "learning_rate": 3.553333333333333e-07,
+      "loss": 0.0012,
+      "reward": 1.8932292461395264,
+      "reward_std": 0.05332084745168686,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8932291269302368,
+      "step": 967
+    },
+    {
+      "completion_length": 120.1875,
+      "epoch": 1.2906666666666666,
+      "grad_norm": 1.058966305492855,
+      "kl": 0.036376953125,
+      "learning_rate": 3.5466666666666667e-07,
+      "loss": 0.0015,
+      "reward": 1.9505208730697632,
+      "reward_std": 0.03366719186306,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9505207538604736,
+      "step": 968
+    },
+    {
+      "completion_length": 133.90625,
+      "epoch": 1.292,
+      "grad_norm": 1.3873791138524048,
+      "kl": 0.036376953125,
+      "learning_rate": 3.5399999999999997e-07,
+      "loss": 0.0015,
+      "reward": 1.7693989276885986,
+      "reward_std": 0.12779967486858368,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8006489276885986,
+      "step": 969
+    },
+    {
+      "completion_length": 136.53125,
+      "epoch": 1.2933333333333334,
+      "grad_norm": 1.601871108764726,
+      "kl": 0.0595703125,
+      "learning_rate": 3.533333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.790034294128418,
+      "reward_std": 0.1303480565547943,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8212843537330627,
+      "step": 970
+    },
+    {
+      "completion_length": 127.140625,
+      "epoch": 1.2946666666666666,
+      "grad_norm": 1.7706196034768433,
+      "kl": 0.041748046875,
+      "learning_rate": 3.526666666666667e-07,
+      "loss": 0.0017,
+      "reward": 1.7838542461395264,
+      "reward_std": 0.1445029228925705,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7994791865348816,
+      "step": 971
+    },
+    {
+      "completion_length": 117.890625,
+      "epoch": 1.296,
+      "grad_norm": 8.616013992095596,
+      "kl": 0.060302734375,
+      "learning_rate": 3.52e-07,
+      "loss": 0.0024,
+      "reward": 1.9375,
+      "reward_std": 0.03474641963839531,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9375,
+      "step": 972
+    },
+    {
+      "completion_length": 141.25,
+      "epoch": 1.2973333333333334,
+      "grad_norm": 8.83316876720644,
+      "kl": 0.08251953125,
+      "learning_rate": 3.5133333333333333e-07,
+      "loss": 0.0033,
+      "reward": 1.664434552192688,
+      "reward_std": 0.23518922924995422,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.695684552192688,
+      "step": 973
+    },
+    {
+      "completion_length": 122.671875,
+      "epoch": 1.2986666666666666,
+      "grad_norm": 2.729805576952169,
+      "kl": 0.057861328125,
+      "learning_rate": 3.506666666666667e-07,
+      "loss": 0.0023,
+      "reward": 1.8377604484558105,
+      "reward_std": 0.06681355834007263,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8377603888511658,
+      "step": 974
+    },
+    {
+      "completion_length": 137.984375,
+      "epoch": 1.3,
+      "grad_norm": 2.8306141124845365,
+      "kl": 0.04638671875,
+      "learning_rate": 3.5e-07,
+      "loss": 0.0019,
+      "reward": 1.8049479722976685,
+      "reward_std": 0.19681575894355774,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8205729722976685,
+      "step": 975
+    },
+    {
+      "completion_length": 122.53125,
+      "epoch": 1.3013333333333335,
+      "grad_norm": 1.2823127155707998,
+      "kl": 0.052978515625,
+      "learning_rate": 3.4933333333333334e-07,
+      "loss": 0.0021,
+      "reward": 1.8776042461395264,
+      "reward_std": 0.0572916641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8776041865348816,
+      "step": 976
+    },
+    {
+      "completion_length": 118.9375,
+      "epoch": 1.3026666666666666,
+      "grad_norm": 1.5396185691181914,
+      "kl": 0.04296875,
+      "learning_rate": 3.4866666666666664e-07,
+      "loss": 0.0017,
+      "reward": 1.9791667461395264,
+      "reward_std": 0.0416666641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9791666865348816,
+      "step": 977
+    },
+    {
+      "completion_length": 125.171875,
+      "epoch": 1.304,
+      "grad_norm": 1.2637172042325284,
+      "kl": 0.0361328125,
+      "learning_rate": 3.4799999999999994e-07,
+      "loss": 0.0014,
+      "reward": 1.8489583730697632,
+      "reward_std": 0.05064382776618004,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8489583730697632,
+      "step": 978
+    },
+    {
+      "completion_length": 138.078125,
+      "epoch": 1.3053333333333335,
+      "grad_norm": 2.896208012292423,
+      "kl": 0.052001953125,
+      "learning_rate": 3.473333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.8645833730697632,
+      "reward_std": 0.04429466277360916,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8645833730697632,
+      "step": 979
+    },
+    {
+      "completion_length": 119.9375,
+      "epoch": 1.3066666666666666,
+      "grad_norm": 3.938175542953266,
+      "kl": 0.05810546875,
+      "learning_rate": 3.4666666666666665e-07,
+      "loss": 0.0023,
+      "reward": 1.8528646230697632,
+      "reward_std": 0.08094505965709686,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8528645038604736,
+      "step": 980
+    },
+    {
+      "completion_length": 127.375,
+      "epoch": 1.308,
+      "grad_norm": 1.9668531014106212,
+      "kl": 0.05517578125,
+      "learning_rate": 3.4599999999999995e-07,
+      "loss": 0.0022,
+      "reward": 1.7511863708496094,
+      "reward_std": 0.14533808827400208,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7668113708496094,
+      "step": 981
+    },
+    {
+      "completion_length": 110.796875,
+      "epoch": 1.3093333333333335,
+      "grad_norm": 0.5715560864585766,
+      "kl": 0.03662109375,
+      "learning_rate": 3.453333333333333e-07,
+      "loss": 0.0015,
+      "reward": 1.9322917461395264,
+      "reward_std": 0.03608439117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9322916269302368,
+      "step": 982
+    },
+    {
+      "completion_length": 111.40625,
+      "epoch": 1.3106666666666666,
+      "grad_norm": 1.2263499939754539,
+      "kl": 0.050537109375,
+      "learning_rate": 3.4466666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.87109375,
+      "reward_std": 0.0703125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.87109375,
+      "step": 983
+    },
+    {
+      "completion_length": 111.234375,
+      "epoch": 1.312,
+      "grad_norm": 1.043514113109278,
+      "kl": 0.0380859375,
+      "learning_rate": 3.4399999999999996e-07,
+      "loss": 0.0015,
+      "reward": 1.9401041269302368,
+      "reward_std": 0.04247239977121353,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9557291269302368,
+      "step": 984
+    },
+    {
+      "completion_length": 122.125,
+      "epoch": 1.3133333333333335,
+      "grad_norm": 1.1996889998489244,
+      "kl": 0.033935546875,
+      "learning_rate": 3.433333333333333e-07,
+      "loss": 0.0014,
+      "reward": 1.754427194595337,
+      "reward_std": 0.07572515308856964,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7544270753860474,
+      "step": 985
+    },
+    {
+      "completion_length": 116.28125,
+      "epoch": 1.3146666666666667,
+      "grad_norm": 1.1852429638828967,
+      "kl": 0.054931640625,
+      "learning_rate": 3.4266666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.9265625476837158,
+      "reward_std": 0.016223199665546417,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9265625476837158,
+      "step": 986
+    },
+    {
+      "completion_length": 128.984375,
+      "epoch": 1.316,
+      "grad_norm": 1.9320005332980406,
+      "kl": 0.06689453125,
+      "learning_rate": 3.42e-07,
+      "loss": 0.0027,
+      "reward": 1.823958396911621,
+      "reward_std": 0.08640611916780472,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8239583969116211,
+      "step": 987
+    },
+    {
+      "completion_length": 114.4375,
+      "epoch": 1.3173333333333335,
+      "grad_norm": 1.2233169539780104,
+      "kl": 0.055419921875,
+      "learning_rate": 3.413333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.8868427276611328,
+      "reward_std": 0.08783789724111557,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9024677276611328,
+      "step": 988
+    },
+    {
+      "completion_length": 117.625,
+      "epoch": 1.3186666666666667,
+      "grad_norm": 2.211138676449477,
+      "kl": 0.047607421875,
+      "learning_rate": 3.4066666666666667e-07,
+      "loss": 0.0019,
+      "reward": 1.818750023841858,
+      "reward_std": 0.15360654890537262,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8187500238418579,
+      "step": 989
+    },
+    {
+      "completion_length": 134.0,
+      "epoch": 1.32,
+      "grad_norm": 2.3900662151577037,
+      "kl": 0.061767578125,
+      "learning_rate": 3.4000000000000003e-07,
+      "loss": 0.0025,
+      "reward": 1.8427083492279053,
+      "reward_std": 0.08059852570295334,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8583333492279053,
+      "step": 990
+    },
+    {
+      "completion_length": 124.671875,
+      "epoch": 1.3213333333333335,
+      "grad_norm": 1.6463866536804814,
+      "kl": 0.04345703125,
+      "learning_rate": 3.3933333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.8463542461395264,
+      "reward_std": 0.1145833283662796,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8463541865348816,
+      "step": 991
+    },
+    {
+      "completion_length": 128.90625,
+      "epoch": 1.3226666666666667,
+      "grad_norm": 2.025710612646314,
+      "kl": 0.08154296875,
+      "learning_rate": 3.386666666666667e-07,
+      "loss": 0.0033,
+      "reward": 1.7901041507720947,
+      "reward_std": 0.1392642855644226,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8213542103767395,
+      "step": 992
+    },
+    {
+      "completion_length": 116.703125,
+      "epoch": 1.324,
+      "grad_norm": 2.6190761237545517,
+      "kl": 0.044921875,
+      "learning_rate": 3.38e-07,
+      "loss": 0.0018,
+      "reward": 1.8125,
+      "reward_std": 0.0416666641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8125,
+      "step": 993
+    },
+    {
+      "completion_length": 128.53125,
+      "epoch": 1.3253333333333333,
+      "grad_norm": 0.30058633813195196,
+      "kl": 0.040771484375,
+      "learning_rate": 3.373333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.8229167461395264,
+      "reward_std": 0.0625,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8385416865348816,
+      "step": 994
+    },
+    {
+      "completion_length": 122.171875,
+      "epoch": 1.3266666666666667,
+      "grad_norm": 1.1396740472207234,
+      "kl": 0.02783203125,
+      "learning_rate": 3.3666666666666664e-07,
+      "loss": 0.0011,
+      "reward": 1.7135417461395264,
+      "reward_std": 0.15625,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7291666865348816,
+      "step": 995
+    },
+    {
+      "completion_length": 117.6875,
+      "epoch": 1.328,
+      "grad_norm": 2.4126663025187103,
+      "kl": 0.0439453125,
+      "learning_rate": 3.36e-07,
+      "loss": 0.0018,
+      "reward": 1.874739646911621,
+      "reward_std": 0.07087166607379913,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8903645873069763,
+      "step": 996
+    },
+    {
+      "completion_length": 126.21875,
+      "epoch": 1.3293333333333333,
+      "grad_norm": 1.809943065061677,
+      "kl": 0.045654296875,
+      "learning_rate": 3.353333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.8723958730697632,
+      "reward_std": 0.06770219653844833,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8723958730697632,
+      "step": 997
+    },
+    {
+      "completion_length": 119.515625,
+      "epoch": 1.3306666666666667,
+      "grad_norm": 1.11307318720347,
+      "kl": 0.05029296875,
+      "learning_rate": 3.3466666666666665e-07,
+      "loss": 0.002,
+      "reward": 1.8429687023162842,
+      "reward_std": 0.08343211561441422,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8429688215255737,
+      "step": 998
+    },
+    {
+      "completion_length": 112.640625,
+      "epoch": 1.332,
+      "grad_norm": 1.354194270806971,
+      "kl": 0.046142578125,
+      "learning_rate": 3.34e-07,
+      "loss": 0.0018,
+      "reward": 1.8796875476837158,
+      "reward_std": 0.04374999925494194,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8796875476837158,
+      "step": 999
+    },
+    {
+      "completion_length": 120.0625,
+      "epoch": 1.3333333333333333,
+      "grad_norm": 1.1601763157593983,
+      "kl": 0.0537109375,
+      "learning_rate": 3.333333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.8984375,
+      "reward_std": 0.11269880086183548,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9140625,
+      "step": 1000
+    },
+    {
+      "completion_length": 111.140625,
+      "epoch": 1.3346666666666667,
+      "grad_norm": 2.259350318609975,
+      "kl": 0.0634765625,
+      "learning_rate": 3.3266666666666665e-07,
+      "loss": 0.0025,
+      "reward": 1.8671875,
+      "reward_std": 0.10277109593153,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8671875,
+      "step": 1001
+    },
+    {
+      "completion_length": 130.546875,
+      "epoch": 1.336,
+      "grad_norm": 1.7270207247689635,
+      "kl": 0.080078125,
+      "learning_rate": 3.32e-07,
+      "loss": 0.0032,
+      "reward": 1.8088542222976685,
+      "reward_std": 0.14872083067893982,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8401041626930237,
+      "step": 1002
+    },
+    {
+      "completion_length": 125.828125,
+      "epoch": 1.3373333333333333,
+      "grad_norm": 3.264071101179222,
+      "kl": 0.045654296875,
+      "learning_rate": 3.313333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.8228050470352173,
+      "reward_std": 0.09443765133619308,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8228050470352173,
+      "step": 1003
+    },
+    {
+      "completion_length": 126.9375,
+      "epoch": 1.3386666666666667,
+      "grad_norm": 2.5021365442343053,
+      "kl": 0.055419921875,
+      "learning_rate": 3.3066666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.8093750476837158,
+      "reward_std": 0.06458333134651184,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8250000476837158,
+      "step": 1004
+    },
+    {
+      "completion_length": 113.625,
+      "epoch": 1.34,
+      "grad_norm": 1.40230328400573,
+      "kl": 0.048095703125,
+      "learning_rate": 3.3e-07,
+      "loss": 0.0019,
+      "reward": 1.7817708253860474,
+      "reward_std": 0.07274993509054184,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7817708849906921,
+      "step": 1005
+    },
+    {
+      "completion_length": 111.8125,
+      "epoch": 1.3413333333333333,
+      "grad_norm": 3.422327455759251,
+      "kl": 0.041259765625,
+      "learning_rate": 3.293333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.8125,
+      "reward_std": 0.06733439117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8125,
+      "step": 1006
+    },
+    {
+      "completion_length": 113.03125,
+      "epoch": 1.3426666666666667,
+      "grad_norm": 2.043919483083014,
+      "kl": 0.0390625,
+      "learning_rate": 3.2866666666666667e-07,
+      "loss": 0.0016,
+      "reward": 1.8802083730697632,
+      "reward_std": 0.0881677195429802,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8802083134651184,
+      "step": 1007
+    },
+    {
+      "completion_length": 115.46875,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 1.3661169402017876,
+      "kl": 0.0303955078125,
+      "learning_rate": 3.28e-07,
+      "loss": 0.0012,
+      "reward": 1.90625,
+      "reward_std": 0.09375,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.90625,
+      "step": 1008
+    },
+    {
+      "completion_length": 117.859375,
+      "epoch": 1.3453333333333333,
+      "grad_norm": 1.4141998887661347,
+      "kl": 0.05859375,
+      "learning_rate": 3.2733333333333327e-07,
+      "loss": 0.0023,
+      "reward": 1.816927194595337,
+      "reward_std": 0.02588011510670185,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8169270753860474,
+      "step": 1009
+    },
+    {
+      "completion_length": 114.78125,
+      "epoch": 1.3466666666666667,
+      "grad_norm": 1.1111945749650376,
+      "kl": 0.04638671875,
+      "learning_rate": 3.2666666666666663e-07,
+      "loss": 0.0019,
+      "reward": 1.8276041746139526,
+      "reward_std": 0.052780430763959885,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8276041746139526,
+      "step": 1010
+    },
+    {
+      "completion_length": 121.109375,
+      "epoch": 1.3479999999999999,
+      "grad_norm": 3.205694155513212,
+      "kl": 0.0693359375,
+      "learning_rate": 3.26e-07,
+      "loss": 0.0028,
+      "reward": 1.6593377590179443,
+      "reward_std": 0.20077505707740784,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6593378782272339,
+      "step": 1011
+    },
+    {
+      "completion_length": 115.9375,
+      "epoch": 1.3493333333333333,
+      "grad_norm": 2.022304390767842,
+      "kl": 0.0634765625,
+      "learning_rate": 3.253333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.7682292461395264,
+      "reward_std": 0.07254272699356079,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7682292461395264,
+      "step": 1012
+    },
+    {
+      "completion_length": 123.75,
+      "epoch": 1.3506666666666667,
+      "grad_norm": 0.9102803274915254,
+      "kl": 0.02294921875,
+      "learning_rate": 3.2466666666666664e-07,
+      "loss": 0.0009,
+      "reward": 1.765625,
+      "reward_std": 0.16591878235340118,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.796875,
+      "step": 1013
+    },
+    {
+      "completion_length": 123.96875,
+      "epoch": 1.3519999999999999,
+      "grad_norm": 2.049396747619361,
+      "kl": 0.068359375,
+      "learning_rate": 3.24e-07,
+      "loss": 0.0027,
+      "reward": 1.7283854484558105,
+      "reward_std": 0.08617881685495377,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7283853888511658,
+      "step": 1014
+    },
+    {
+      "completion_length": 130.5,
+      "epoch": 1.3533333333333333,
+      "grad_norm": 6.101723146618242,
+      "kl": 0.06298828125,
+      "learning_rate": 3.233333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.7135416269302368,
+      "reward_std": 0.11224675178527832,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7135416269302368,
+      "step": 1015
+    },
+    {
+      "completion_length": 111.1875,
+      "epoch": 1.3546666666666667,
+      "grad_norm": 2.9234070791906492,
+      "kl": 0.054931640625,
+      "learning_rate": 3.2266666666666664e-07,
+      "loss": 0.0022,
+      "reward": 1.7050223350524902,
+      "reward_std": 0.10876703262329102,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7050223350524902,
+      "step": 1016
+    },
+    {
+      "completion_length": 114.03125,
+      "epoch": 1.3559999999999999,
+      "grad_norm": 1.2361574719475896,
+      "kl": 0.052490234375,
+      "learning_rate": 3.22e-07,
+      "loss": 0.0021,
+      "reward": 1.8541667461395264,
+      "reward_std": 0.0416666641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8541666865348816,
+      "step": 1017
+    },
+    {
+      "completion_length": 116.59375,
+      "epoch": 1.3573333333333333,
+      "grad_norm": 1.4267887947596953,
+      "kl": 0.05859375,
+      "learning_rate": 3.2133333333333335e-07,
+      "loss": 0.0023,
+      "reward": 1.9140625,
+      "reward_std": 0.049882031977176666,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9140625596046448,
+      "step": 1018
+    },
+    {
+      "completion_length": 119.125,
+      "epoch": 1.3586666666666667,
+      "grad_norm": 0.9083612305273092,
+      "kl": 0.02587890625,
+      "learning_rate": 3.2066666666666665e-07,
+      "loss": 0.001,
+      "reward": 1.96875,
+      "reward_std": 0.0625,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.984375,
+      "step": 1019
+    },
+    {
+      "completion_length": 116.09375,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 1.9638689832682292,
+      "kl": 0.059326171875,
+      "learning_rate": 3.2e-07,
+      "loss": 0.0024,
+      "reward": 1.9236979484558105,
+      "reward_std": 0.06473022699356079,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9236979484558105,
+      "step": 1020
+    },
+    {
+      "completion_length": 115.046875,
+      "epoch": 1.3613333333333333,
+      "grad_norm": 3.5617768378771726,
+      "kl": 0.054443359375,
+      "learning_rate": 3.1933333333333336e-07,
+      "loss": 0.0022,
+      "reward": 1.926302194595337,
+      "reward_std": 0.06385204195976257,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9263021349906921,
+      "step": 1021
+    },
+    {
+      "completion_length": 123.84375,
+      "epoch": 1.3626666666666667,
+      "grad_norm": 1.7169289646263977,
+      "kl": 0.061767578125,
+      "learning_rate": 3.1866666666666666e-07,
+      "loss": 0.0025,
+      "reward": 1.8110491037368774,
+      "reward_std": 0.19939221441745758,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8266741037368774,
+      "step": 1022
+    },
+    {
+      "completion_length": 120.84375,
+      "epoch": 1.3639999999999999,
+      "grad_norm": 5.629321418171578,
+      "kl": 0.068359375,
+      "learning_rate": 3.18e-07,
+      "loss": 0.0027,
+      "reward": 1.7804992198944092,
+      "reward_std": 0.167941614985466,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7961242198944092,
+      "step": 1023
+    },
+    {
+      "completion_length": 121.796875,
+      "epoch": 1.3653333333333333,
+      "grad_norm": 1.0374212032273207,
+      "kl": 0.042724609375,
+      "learning_rate": 3.173333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.96484375,
+      "reward_std": 0.0286458320915699,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.96484375,
+      "step": 1024
+    },
+    {
+      "completion_length": 118.578125,
+      "epoch": 1.3666666666666667,
+      "grad_norm": 1.301841466670957,
+      "kl": 0.055908203125,
+      "learning_rate": 3.166666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.8796875476837158,
+      "reward_std": 0.12142626196146011,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.879687488079071,
+      "step": 1025
+    },
+    {
+      "completion_length": 122.015625,
+      "epoch": 1.3679999999999999,
+      "grad_norm": 1.485154990894895,
+      "kl": 0.0439453125,
+      "learning_rate": 3.1599999999999997e-07,
+      "loss": 0.0018,
+      "reward": 1.9244792461395264,
+      "reward_std": 0.1246260553598404,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.9557291865348816,
+      "step": 1026
+    },
+    {
+      "completion_length": 107.171875,
+      "epoch": 1.3693333333333333,
+      "grad_norm": 1.5988765539182854,
+      "kl": 0.025634765625,
+      "learning_rate": 3.153333333333333e-07,
+      "loss": 0.001,
+      "reward": 1.8125,
+      "reward_std": 0.09999999403953552,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8125,
+      "step": 1027
+    },
+    {
+      "completion_length": 110.859375,
+      "epoch": 1.3706666666666667,
+      "grad_norm": 7.26546557802646,
+      "kl": 0.03369140625,
+      "learning_rate": 3.146666666666666e-07,
+      "loss": 0.0013,
+      "reward": 1.8546874523162842,
+      "reward_std": 0.0054126582108438015,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.854687511920929,
+      "step": 1028
+    },
+    {
+      "completion_length": 123.78125,
+      "epoch": 1.3719999999999999,
+      "grad_norm": 2.182324371104986,
+      "kl": 0.06787109375,
+      "learning_rate": 3.14e-07,
+      "loss": 0.0027,
+      "reward": 1.8427083492279053,
+      "reward_std": 0.07424470037221909,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8427083492279053,
+      "step": 1029
+    },
+    {
+      "completion_length": 116.390625,
+      "epoch": 1.3733333333333333,
+      "grad_norm": 1.7811669127142826,
+      "kl": 0.068359375,
+      "learning_rate": 3.1333333333333333e-07,
+      "loss": 0.0027,
+      "reward": 1.8859374523162842,
+      "reward_std": 0.08742521703243256,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.885937511920929,
+      "step": 1030
+    },
+    {
+      "completion_length": 106.734375,
+      "epoch": 1.3746666666666667,
+      "grad_norm": 1.6590966164355774,
+      "kl": 0.0625,
+      "learning_rate": 3.1266666666666663e-07,
+      "loss": 0.0025,
+      "reward": 1.8333333730697632,
+      "reward_std": 0.0624999925494194,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8333333730697632,
+      "step": 1031
+    },
+    {
+      "completion_length": 112.125,
+      "epoch": 1.376,
+      "grad_norm": 4.475657961952698,
+      "kl": 0.07861328125,
+      "learning_rate": 3.12e-07,
+      "loss": 0.0031,
+      "reward": 1.90234375,
+      "reward_std": 0.09495859593153,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.90234375,
+      "step": 1032
+    },
+    {
+      "completion_length": 121.84375,
+      "epoch": 1.3773333333333333,
+      "grad_norm": 1.5223146452668734,
+      "kl": 0.03955078125,
+      "learning_rate": 3.1133333333333334e-07,
+      "loss": 0.0016,
+      "reward": 1.8697917461395264,
+      "reward_std": 0.12059739232063293,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.9010416865348816,
+      "step": 1033
+    },
+    {
+      "completion_length": 114.046875,
+      "epoch": 1.3786666666666667,
+      "grad_norm": 25.108242720443762,
+      "kl": 0.048095703125,
+      "learning_rate": 3.1066666666666664e-07,
+      "loss": 0.0019,
+      "reward": 1.7552083730697632,
+      "reward_std": 0.043278127908706665,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7552083730697632,
+      "step": 1034
+    },
+    {
+      "completion_length": 126.34375,
+      "epoch": 1.38,
+      "grad_norm": 1.385093674506436,
+      "kl": 0.052734375,
+      "learning_rate": 3.1e-07,
+      "loss": 0.0021,
+      "reward": 1.7476563453674316,
+      "reward_std": 0.12059885263442993,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.8101562261581421,
+      "step": 1035
+    },
+    {
+      "completion_length": 112.609375,
+      "epoch": 1.3813333333333333,
+      "grad_norm": 2.2117907377130663,
+      "kl": 0.0478515625,
+      "learning_rate": 3.0933333333333335e-07,
+      "loss": 0.0019,
+      "reward": 1.8848958015441895,
+      "reward_std": 0.0550716407597065,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9005208611488342,
+      "step": 1036
+    },
+    {
+      "completion_length": 116.046875,
+      "epoch": 1.3826666666666667,
+      "grad_norm": 6.168362589554847,
+      "kl": 0.0400390625,
+      "learning_rate": 3.0866666666666665e-07,
+      "loss": 0.0016,
+      "reward": 1.8489583730697632,
+      "reward_std": 0.06411145627498627,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8489583730697632,
+      "step": 1037
+    },
+    {
+      "completion_length": 134.109375,
+      "epoch": 1.384,
+      "grad_norm": 1.9290611362719066,
+      "kl": 0.07421875,
+      "learning_rate": 3.08e-07,
+      "loss": 0.003,
+      "reward": 1.8674479722976685,
+      "reward_std": 0.0996750220656395,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8674479722976685,
+      "step": 1038
+    },
+    {
+      "completion_length": 119.9375,
+      "epoch": 1.3853333333333333,
+      "grad_norm": 2.0031951221644535,
+      "kl": 0.05908203125,
+      "learning_rate": 3.0733333333333336e-07,
+      "loss": 0.0024,
+      "reward": 1.8294271230697632,
+      "reward_std": 0.10639689117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8294271230697632,
+      "step": 1039
+    },
+    {
+      "completion_length": 114.5,
+      "epoch": 1.3866666666666667,
+      "grad_norm": 1.7927659690543012,
+      "kl": 0.07568359375,
+      "learning_rate": 3.066666666666666e-07,
+      "loss": 0.003,
+      "reward": 1.808333396911621,
+      "reward_std": 0.053644463419914246,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8083333373069763,
+      "step": 1040
+    },
+    {
+      "completion_length": 111.09375,
+      "epoch": 1.388,
+      "grad_norm": 1.7071744599060792,
+      "kl": 0.048828125,
+      "learning_rate": 3.0599999999999996e-07,
+      "loss": 0.0019,
+      "reward": 1.8414063453674316,
+      "reward_std": 0.07519235461950302,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8414062857627869,
+      "step": 1041
+    },
+    {
+      "completion_length": 118.875,
+      "epoch": 1.3893333333333333,
+      "grad_norm": 2.7861769243396663,
+      "kl": 0.04541015625,
+      "learning_rate": 3.053333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.6796875,
+      "reward_std": 0.17670938372612,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6796875,
+      "step": 1042
+    },
+    {
+      "completion_length": 119.421875,
+      "epoch": 1.3906666666666667,
+      "grad_norm": 2.011943651236363,
+      "kl": 0.04052734375,
+      "learning_rate": 3.046666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.7453868389129639,
+      "reward_std": 0.12336866557598114,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7610118985176086,
+      "step": 1043
+    },
+    {
+      "completion_length": 111.921875,
+      "epoch": 1.392,
+      "grad_norm": 1.4202651411450509,
+      "kl": 0.06982421875,
+      "learning_rate": 3.0399999999999997e-07,
+      "loss": 0.0028,
+      "reward": 1.863541603088379,
+      "reward_std": 0.06574606150388718,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8635417222976685,
+      "step": 1044
+    },
+    {
+      "completion_length": 131.515625,
+      "epoch": 1.3933333333333333,
+      "grad_norm": 1.0054859241247873,
+      "kl": 0.036865234375,
+      "learning_rate": 3.033333333333333e-07,
+      "loss": 0.0015,
+      "reward": 1.8723958730697632,
+      "reward_std": 0.0989583283662796,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8723958730697632,
+      "step": 1045
+    },
+    {
+      "completion_length": 122.0,
+      "epoch": 1.3946666666666667,
+      "grad_norm": 1.578927452451213,
+      "kl": 0.0439453125,
+      "learning_rate": 3.026666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.828125,
+      "reward_std": 0.13200798630714417,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.859375,
+      "step": 1046
+    },
+    {
+      "completion_length": 128.140625,
+      "epoch": 1.396,
+      "grad_norm": 3.602949334975548,
+      "kl": 0.05322265625,
+      "learning_rate": 3.02e-07,
+      "loss": 0.0021,
+      "reward": 1.8697917461395264,
+      "reward_std": 0.16026698052883148,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8854166865348816,
+      "step": 1047
+    },
+    {
+      "completion_length": 108.34375,
+      "epoch": 1.3973333333333333,
+      "grad_norm": 2.053333652581538,
+      "kl": 0.05810546875,
+      "learning_rate": 3.0133333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.9307291507720947,
+      "reward_std": 0.06145832687616348,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9307291507720947,
+      "step": 1048
+    },
+    {
+      "completion_length": 133.1875,
+      "epoch": 1.3986666666666667,
+      "grad_norm": 1.5557195644507582,
+      "kl": 0.0537109375,
+      "learning_rate": 3.006666666666667e-07,
+      "loss": 0.0021,
+      "reward": 1.7390625476837158,
+      "reward_std": 0.14821045100688934,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7703125476837158,
+      "step": 1049
+    },
+    {
+      "completion_length": 128.078125,
+      "epoch": 1.4,
+      "grad_norm": 1.3387608008440295,
+      "kl": 0.052490234375,
+      "learning_rate": 3e-07,
+      "loss": 0.0021,
+      "reward": 1.935819149017334,
+      "reward_std": 0.08669507503509521,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9514442086219788,
+      "step": 1050
+    },
+    {
+      "completion_length": 114.34375,
+      "epoch": 1.4013333333333333,
+      "grad_norm": 2.9561716342513815,
+      "kl": 0.0595703125,
+      "learning_rate": 2.9933333333333334e-07,
+      "loss": 0.0024,
+      "reward": 1.7698661088943481,
+      "reward_std": 0.18023674190044403,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7854911088943481,
+      "step": 1051
+    },
+    {
+      "completion_length": 124.421875,
+      "epoch": 1.4026666666666667,
+      "grad_norm": 1.2601868852206264,
+      "kl": 0.03515625,
+      "learning_rate": 2.986666666666667e-07,
+      "loss": 0.0014,
+      "reward": 1.8802083730697632,
+      "reward_std": 0.09021097421646118,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8802083730697632,
+      "step": 1052
+    },
+    {
+      "completion_length": 112.109375,
+      "epoch": 1.404,
+      "grad_norm": 1.6648077566315551,
+      "kl": 0.05029296875,
+      "learning_rate": 2.98e-07,
+      "loss": 0.002,
+      "reward": 1.899999976158142,
+      "reward_std": 0.04895832762122154,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9000000357627869,
+      "step": 1053
+    },
+    {
+      "completion_length": 112.15625,
+      "epoch": 1.4053333333333333,
+      "grad_norm": 1.674475351894077,
+      "kl": 0.058349609375,
+      "learning_rate": 2.9733333333333335e-07,
+      "loss": 0.0023,
+      "reward": 1.926041603088379,
+      "reward_std": 0.07528164237737656,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9416667222976685,
+      "step": 1054
+    },
+    {
+      "completion_length": 119.25,
+      "epoch": 1.4066666666666667,
+      "grad_norm": 1.6775866190518116,
+      "kl": 0.059814453125,
+      "learning_rate": 2.966666666666667e-07,
+      "loss": 0.0024,
+      "reward": 1.7693452835083008,
+      "reward_std": 0.06769795715808868,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7693452835083008,
+      "step": 1055
+    },
+    {
+      "completion_length": 120.75,
+      "epoch": 1.408,
+      "grad_norm": 1.9996998089879137,
+      "kl": 0.064453125,
+      "learning_rate": 2.9599999999999995e-07,
+      "loss": 0.0026,
+      "reward": 1.8182291984558105,
+      "reward_std": 0.10428772866725922,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8182291984558105,
+      "step": 1056
+    },
+    {
+      "completion_length": 122.78125,
+      "epoch": 1.4093333333333333,
+      "grad_norm": 1.513144681265203,
+      "kl": 0.0301513671875,
+      "learning_rate": 2.953333333333333e-07,
+      "loss": 0.0012,
+      "reward": 1.7531249523162842,
+      "reward_std": 0.19374999403953552,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7531249523162842,
+      "step": 1057
+    },
+    {
+      "completion_length": 119.21875,
+      "epoch": 1.4106666666666667,
+      "grad_norm": 5.17545501575887,
+      "kl": 0.07861328125,
+      "learning_rate": 2.9466666666666666e-07,
+      "loss": 0.0031,
+      "reward": 1.91796875,
+      "reward_std": 0.041756417602300644,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.91796875,
+      "step": 1058
+    },
+    {
+      "completion_length": 114.1875,
+      "epoch": 1.412,
+      "grad_norm": 1.2083105705119042,
+      "kl": 0.042724609375,
+      "learning_rate": 2.9399999999999996e-07,
+      "loss": 0.0017,
+      "reward": 1.921875,
+      "reward_std": 0.018042195588350296,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.921875,
+      "step": 1059
+    },
+    {
+      "completion_length": 112.8125,
+      "epoch": 1.4133333333333333,
+      "grad_norm": 2.0165359108647025,
+      "kl": 0.06689453125,
+      "learning_rate": 2.933333333333333e-07,
+      "loss": 0.0027,
+      "reward": 1.81640625,
+      "reward_std": 0.09750150144100189,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.81640625,
+      "step": 1060
+    },
+    {
+      "completion_length": 117.890625,
+      "epoch": 1.4146666666666667,
+      "grad_norm": 1.8328544325261384,
+      "kl": 0.051025390625,
+      "learning_rate": 2.9266666666666667e-07,
+      "loss": 0.002,
+      "reward": 1.7572916746139526,
+      "reward_std": 0.18207451701164246,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7572916746139526,
+      "step": 1061
+    },
+    {
+      "completion_length": 118.671875,
+      "epoch": 1.416,
+      "grad_norm": 1.7602793311789342,
+      "kl": 0.049560546875,
+      "learning_rate": 2.9199999999999997e-07,
+      "loss": 0.002,
+      "reward": 1.8213541507720947,
+      "reward_std": 0.15602411329746246,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8213542103767395,
+      "step": 1062
+    },
+    {
+      "completion_length": 103.28125,
+      "epoch": 1.4173333333333333,
+      "grad_norm": 1.3978369393831434,
+      "kl": 0.058349609375,
+      "learning_rate": 2.913333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.9500000476837158,
+      "reward_std": 0.01666666939854622,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.949999988079071,
+      "step": 1063
+    },
+    {
+      "completion_length": 109.4375,
+      "epoch": 1.4186666666666667,
+      "grad_norm": 1.9968997442757348,
+      "kl": 0.07373046875,
+      "learning_rate": 2.906666666666667e-07,
+      "loss": 0.003,
+      "reward": 1.873437523841858,
+      "reward_std": 0.06145833432674408,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8734375238418579,
+      "step": 1064
+    },
+    {
+      "completion_length": 116.734375,
+      "epoch": 1.42,
+      "grad_norm": 1.6418702107513092,
+      "kl": 0.052978515625,
+      "learning_rate": 2.9e-07,
+      "loss": 0.0021,
+      "reward": 1.8718750476837158,
+      "reward_std": 0.09304219484329224,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8718750476837158,
+      "step": 1065
+    },
+    {
+      "completion_length": 114.546875,
+      "epoch": 1.4213333333333333,
+      "grad_norm": 1.6949918842212757,
+      "kl": 0.0419921875,
+      "learning_rate": 2.8933333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.9349702596664429,
+      "reward_std": 0.01280956994742155,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9349702596664429,
+      "step": 1066
+    },
+    {
+      "completion_length": 104.859375,
+      "epoch": 1.4226666666666667,
+      "grad_norm": 1.4572170146028594,
+      "kl": 0.0625,
+      "learning_rate": 2.886666666666667e-07,
+      "loss": 0.0025,
+      "reward": 1.860937476158142,
+      "reward_std": 0.046769775450229645,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8609375357627869,
+      "step": 1067
+    },
+    {
+      "completion_length": 115.859375,
+      "epoch": 1.424,
+      "grad_norm": 2.568084230074981,
+      "kl": 0.0419921875,
+      "learning_rate": 2.88e-07,
+      "loss": 0.0017,
+      "reward": 1.8382575511932373,
+      "reward_std": 0.14829355478286743,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8382575511932373,
+      "step": 1068
+    },
+    {
+      "completion_length": 109.875,
+      "epoch": 1.4253333333333333,
+      "grad_norm": 1.1551057140079328,
+      "kl": 0.060546875,
+      "learning_rate": 2.8733333333333334e-07,
+      "loss": 0.0024,
+      "reward": 1.902604103088379,
+      "reward_std": 0.024263188242912292,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9026041030883789,
+      "step": 1069
+    },
+    {
+      "completion_length": 117.8125,
+      "epoch": 1.4266666666666667,
+      "grad_norm": 1.839339991266919,
+      "kl": 0.034912109375,
+      "learning_rate": 2.866666666666667e-07,
+      "loss": 0.0014,
+      "reward": 1.7591146230697632,
+      "reward_std": 0.08341985195875168,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7591146230697632,
+      "step": 1070
+    },
+    {
+      "completion_length": 129.28125,
+      "epoch": 1.428,
+      "grad_norm": 1.0266279423558735,
+      "kl": 0.05419921875,
+      "learning_rate": 2.8599999999999994e-07,
+      "loss": 0.0022,
+      "reward": 1.8385417461395264,
+      "reward_std": 0.04650105535984039,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8385416865348816,
+      "step": 1071
+    },
+    {
+      "completion_length": 113.53125,
+      "epoch": 1.4293333333333333,
+      "grad_norm": 1.2454058130898156,
+      "kl": 0.04736328125,
+      "learning_rate": 2.853333333333333e-07,
+      "loss": 0.0019,
+      "reward": 1.943750023841858,
+      "reward_std": 0.05300947278738022,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9437500238418579,
+      "step": 1072
+    },
+    {
+      "completion_length": 118.828125,
+      "epoch": 1.4306666666666668,
+      "grad_norm": 1.4323934740036783,
+      "kl": 0.042236328125,
+      "learning_rate": 2.8466666666666665e-07,
+      "loss": 0.0017,
+      "reward": 1.8179688453674316,
+      "reward_std": 0.0844239592552185,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8335937857627869,
+      "step": 1073
+    },
+    {
+      "completion_length": 116.578125,
+      "epoch": 1.432,
+      "grad_norm": 1.644296904389035,
+      "kl": 0.032470703125,
+      "learning_rate": 2.8399999999999995e-07,
+      "loss": 0.0013,
+      "reward": 1.7572916746139526,
+      "reward_std": 0.11723941564559937,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7729166746139526,
+      "step": 1074
+    },
+    {
+      "completion_length": 115.671875,
+      "epoch": 1.4333333333333333,
+      "grad_norm": 2.824390852511179,
+      "kl": 0.08984375,
+      "learning_rate": 2.833333333333333e-07,
+      "loss": 0.0036,
+      "reward": 1.9302828311920166,
+      "reward_std": 0.02983361855149269,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.930282711982727,
+      "step": 1075
+    },
+    {
+      "completion_length": 122.046875,
+      "epoch": 1.4346666666666668,
+      "grad_norm": 1.7676747914374529,
+      "kl": 0.0556640625,
+      "learning_rate": 2.8266666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.8026041984558105,
+      "reward_std": 0.08355759084224701,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8026041984558105,
+      "step": 1076
+    },
+    {
+      "completion_length": 117.078125,
+      "epoch": 1.436,
+      "grad_norm": 1.1150376298985358,
+      "kl": 0.038818359375,
+      "learning_rate": 2.8199999999999996e-07,
+      "loss": 0.0016,
+      "reward": 1.7408483028411865,
+      "reward_std": 0.09962436556816101,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7408482432365417,
+      "step": 1077
+    },
+    {
+      "completion_length": 115.46875,
+      "epoch": 1.4373333333333334,
+      "grad_norm": 2.2823610135567773,
+      "kl": 0.06298828125,
+      "learning_rate": 2.813333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.9265624284744263,
+      "reward_std": 0.12639866769313812,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9265625476837158,
+      "step": 1078
+    },
+    {
+      "completion_length": 107.046875,
+      "epoch": 1.4386666666666668,
+      "grad_norm": 1.7400488001685523,
+      "kl": 0.052978515625,
+      "learning_rate": 2.8066666666666667e-07,
+      "loss": 0.0021,
+      "reward": 1.8098958730697632,
+      "reward_std": 0.0989583283662796,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8255208730697632,
+      "step": 1079
+    },
+    {
+      "completion_length": 116.9375,
+      "epoch": 1.44,
+      "grad_norm": 1.9866543923053557,
+      "kl": 0.058837890625,
+      "learning_rate": 2.8e-07,
+      "loss": 0.0024,
+      "reward": 1.8580729961395264,
+      "reward_std": 0.07504989951848984,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8580729365348816,
+      "step": 1080
+    },
+    {
+      "completion_length": 121.90625,
+      "epoch": 1.4413333333333334,
+      "grad_norm": 1.8438678619371685,
+      "kl": 0.059814453125,
+      "learning_rate": 2.793333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.8458333015441895,
+      "reward_std": 0.12287658452987671,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8614583611488342,
+      "step": 1081
+    },
+    {
+      "completion_length": 114.640625,
+      "epoch": 1.4426666666666668,
+      "grad_norm": 0.8533164898486268,
+      "kl": 0.03662109375,
+      "learning_rate": 2.786666666666667e-07,
+      "loss": 0.0015,
+      "reward": 1.8177083730697632,
+      "reward_std": 0.0625,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8177083730697632,
+      "step": 1082
+    },
+    {
+      "completion_length": 114.0625,
+      "epoch": 1.444,
+      "grad_norm": 1.2423075781449107,
+      "kl": 0.053466796875,
+      "learning_rate": 2.7800000000000003e-07,
+      "loss": 0.0021,
+      "reward": 1.875,
+      "reward_std": 0.05809739604592323,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.875,
+      "step": 1083
+    },
+    {
+      "completion_length": 126.546875,
+      "epoch": 1.4453333333333334,
+      "grad_norm": 1.4283735113997242,
+      "kl": 0.044189453125,
+      "learning_rate": 2.7733333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.8447916507720947,
+      "reward_std": 0.0794244259595871,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8447916507720947,
+      "step": 1084
+    },
+    {
+      "completion_length": 117.21875,
+      "epoch": 1.4466666666666668,
+      "grad_norm": 1.3642681333938804,
+      "kl": 0.05322265625,
+      "learning_rate": 2.766666666666667e-07,
+      "loss": 0.0021,
+      "reward": 1.797842264175415,
+      "reward_std": 0.0901932418346405,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8134673237800598,
+      "step": 1085
+    },
+    {
+      "completion_length": 121.5625,
+      "epoch": 1.448,
+      "grad_norm": 2.316303114323313,
+      "kl": 0.060791015625,
+      "learning_rate": 2.7600000000000004e-07,
+      "loss": 0.0024,
+      "reward": 1.8259673118591309,
+      "reward_std": 0.07656985521316528,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8259673118591309,
+      "step": 1086
+    },
+    {
+      "completion_length": 116.515625,
+      "epoch": 1.4493333333333334,
+      "grad_norm": 1.2290221545257134,
+      "kl": 0.02783203125,
+      "learning_rate": 2.753333333333333e-07,
+      "loss": 0.0011,
+      "reward": 1.9661458730697632,
+      "reward_std": 0.0364583283662796,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9661458730697632,
+      "step": 1087
+    },
+    {
+      "completion_length": 123.921875,
+      "epoch": 1.4506666666666668,
+      "grad_norm": 2.10064725984503,
+      "kl": 0.0615234375,
+      "learning_rate": 2.7466666666666664e-07,
+      "loss": 0.0025,
+      "reward": 1.8781249523162842,
+      "reward_std": 0.11041666567325592,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.893750011920929,
+      "step": 1088
+    },
+    {
+      "completion_length": 121.609375,
+      "epoch": 1.452,
+      "grad_norm": 1.159317210051619,
+      "kl": 0.0458984375,
+      "learning_rate": 2.74e-07,
+      "loss": 0.0018,
+      "reward": 1.9018973112106323,
+      "reward_std": 0.09041955322027206,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9175223112106323,
+      "step": 1089
+    },
+    {
+      "completion_length": 117.609375,
+      "epoch": 1.4533333333333334,
+      "grad_norm": 2.6582886997453565,
+      "kl": 0.04443359375,
+      "learning_rate": 2.733333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.9017857313156128,
+      "reward_std": 0.0904272049665451,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9017857313156128,
+      "step": 1090
+    },
+    {
+      "completion_length": 119.046875,
+      "epoch": 1.4546666666666668,
+      "grad_norm": 0.5178916499620764,
+      "kl": 0.049072265625,
+      "learning_rate": 2.7266666666666665e-07,
+      "loss": 0.002,
+      "reward": 1.9479167461395264,
+      "reward_std": 0.03608439117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9479166865348816,
+      "step": 1091
+    },
+    {
+      "completion_length": 125.171875,
+      "epoch": 1.456,
+      "grad_norm": 1.2344902256110237,
+      "kl": 0.045654296875,
+      "learning_rate": 2.72e-07,
+      "loss": 0.0018,
+      "reward": 1.7999999523162842,
+      "reward_std": 0.07358439266681671,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7999999523162842,
+      "step": 1092
+    },
+    {
+      "completion_length": 119.59375,
+      "epoch": 1.4573333333333334,
+      "grad_norm": 2.0879149579895224,
+      "kl": 0.05615234375,
+      "learning_rate": 2.713333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.8210937976837158,
+      "reward_std": 0.05739115923643112,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.821093738079071,
+      "step": 1093
+    },
+    {
+      "completion_length": 111.859375,
+      "epoch": 1.4586666666666668,
+      "grad_norm": 1.6431315785621254,
+      "kl": 0.052490234375,
+      "learning_rate": 2.7066666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.8671875,
+      "reward_std": 0.053254205733537674,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8671874403953552,
+      "step": 1094
+    },
+    {
+      "completion_length": 121.296875,
+      "epoch": 1.46,
+      "grad_norm": 1.362482124546802,
+      "kl": 0.046142578125,
+      "learning_rate": 2.7e-07,
+      "loss": 0.0018,
+      "reward": 1.8984375,
+      "reward_std": 0.06139424070715904,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8984375,
+      "step": 1095
+    },
+    {
+      "completion_length": 123.59375,
+      "epoch": 1.4613333333333334,
+      "grad_norm": 3.200080671794016,
+      "kl": 0.02734375,
+      "learning_rate": 2.693333333333333e-07,
+      "loss": 0.0011,
+      "reward": 1.9541666507720947,
+      "reward_std": 0.0556742325425148,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9541666507720947,
+      "step": 1096
+    },
+    {
+      "completion_length": 113.0625,
+      "epoch": 1.4626666666666668,
+      "grad_norm": 1.613684754726999,
+      "kl": 0.043212890625,
+      "learning_rate": 2.6866666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.8284116983413696,
+      "reward_std": 0.10118239372968674,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8440366983413696,
+      "step": 1097
+    },
+    {
+      "completion_length": 121.5,
+      "epoch": 1.464,
+      "grad_norm": 8.567233575091818,
+      "kl": 0.04150390625,
+      "learning_rate": 2.68e-07,
+      "loss": 0.0017,
+      "reward": 1.8666666746139526,
+      "reward_std": 0.13280409574508667,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8666666746139526,
+      "step": 1098
+    },
+    {
+      "completion_length": 118.1875,
+      "epoch": 1.4653333333333334,
+      "grad_norm": 1.2940043150450256,
+      "kl": 0.039794921875,
+      "learning_rate": 2.673333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.700781226158142,
+      "reward_std": 0.10327189415693283,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7164062261581421,
+      "step": 1099
+    },
+    {
+      "completion_length": 129.390625,
+      "epoch": 1.4666666666666668,
+      "grad_norm": 2.0659722566841334,
+      "kl": 0.06494140625,
+      "learning_rate": 2.6666666666666667e-07,
+      "loss": 0.0026,
+      "reward": 1.6796875,
+      "reward_std": 0.24167433381080627,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7421875,
+      "step": 1100
+    },
+    {
+      "completion_length": 107.40625,
+      "epoch": 1.468,
+      "grad_norm": 0.956629623364733,
+      "kl": 0.052490234375,
+      "learning_rate": 2.66e-07,
+      "loss": 0.0021,
+      "reward": 1.875,
+      "reward_std": 0.0625,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.875,
+      "step": 1101
+    },
+    {
+      "completion_length": 119.46875,
+      "epoch": 1.4693333333333334,
+      "grad_norm": 1.054575366825908,
+      "kl": 0.04345703125,
+      "learning_rate": 2.653333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.919531226158142,
+      "reward_std": 0.006665600463747978,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9195312261581421,
+      "step": 1102
+    },
+    {
+      "completion_length": 127.734375,
+      "epoch": 1.4706666666666668,
+      "grad_norm": 2.0817573336403545,
+      "kl": 0.06787109375,
+      "learning_rate": 2.6466666666666663e-07,
+      "loss": 0.0027,
+      "reward": 1.6876115798950195,
+      "reward_std": 0.14680279791355133,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7032365798950195,
+      "step": 1103
+    },
+    {
+      "completion_length": 122.796875,
+      "epoch": 1.472,
+      "grad_norm": 1.622786396265799,
+      "kl": 0.048095703125,
+      "learning_rate": 2.64e-07,
+      "loss": 0.0019,
+      "reward": 1.8515625,
+      "reward_std": 0.08763547986745834,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8515625,
+      "step": 1104
+    },
+    {
+      "completion_length": 107.828125,
+      "epoch": 1.4733333333333334,
+      "grad_norm": 1.7237912955351054,
+      "kl": 0.049072265625,
+      "learning_rate": 2.633333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.8958333730697632,
+      "reward_std": 0.11383544653654099,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8958333730697632,
+      "step": 1105
+    },
+    {
+      "completion_length": 120.125,
+      "epoch": 1.4746666666666668,
+      "grad_norm": 5.400669513107469,
+      "kl": 0.061767578125,
+      "learning_rate": 2.6266666666666664e-07,
+      "loss": 0.0025,
+      "reward": 1.8328125476837158,
+      "reward_std": 0.029037879779934883,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8328125476837158,
+      "step": 1106
+    },
+    {
+      "completion_length": 112.3125,
+      "epoch": 1.476,
+      "grad_norm": 2.3772682570322745,
+      "kl": 0.044921875,
+      "learning_rate": 2.62e-07,
+      "loss": 0.0018,
+      "reward": 1.7721354961395264,
+      "reward_std": 0.07514689117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7721354365348816,
+      "step": 1107
+    },
+    {
+      "completion_length": 113.90625,
+      "epoch": 1.4773333333333334,
+      "grad_norm": 1.4855665197036534,
+      "kl": 0.055419921875,
+      "learning_rate": 2.613333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.8315104246139526,
+      "reward_std": 0.07760415971279144,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8471354246139526,
+      "step": 1108
+    },
+    {
+      "completion_length": 122.171875,
+      "epoch": 1.4786666666666668,
+      "grad_norm": 2.0837657844951902,
+      "kl": 0.05908203125,
+      "learning_rate": 2.6066666666666664e-07,
+      "loss": 0.0024,
+      "reward": 1.7236979007720947,
+      "reward_std": 0.12135416269302368,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.7861979007720947,
+      "step": 1109
+    },
+    {
+      "completion_length": 119.6875,
+      "epoch": 1.48,
+      "grad_norm": 2.717095986156113,
+      "kl": 0.0712890625,
+      "learning_rate": 2.6e-07,
+      "loss": 0.0028,
+      "reward": 1.8375000953674316,
+      "reward_std": 0.07302876561880112,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8374999761581421,
+      "step": 1110
+    },
+    {
+      "completion_length": 112.734375,
+      "epoch": 1.4813333333333334,
+      "grad_norm": 2.0183113331450064,
+      "kl": 0.0849609375,
+      "learning_rate": 2.5933333333333335e-07,
+      "loss": 0.0034,
+      "reward": 1.8354166746139526,
+      "reward_std": 0.06458333134651184,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8354166150093079,
+      "step": 1111
+    },
+    {
+      "completion_length": 123.65625,
+      "epoch": 1.4826666666666668,
+      "grad_norm": 0.9147886073722504,
+      "kl": 0.038330078125,
+      "learning_rate": 2.5866666666666665e-07,
+      "loss": 0.0015,
+      "reward": 1.9609375,
+      "reward_std": 0.05170939117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9609375,
+      "step": 1112
+    },
+    {
+      "completion_length": 124.109375,
+      "epoch": 1.484,
+      "grad_norm": 1.5239308538420255,
+      "kl": 0.06884765625,
+      "learning_rate": 2.58e-07,
+      "loss": 0.0027,
+      "reward": 1.8156249523162842,
+      "reward_std": 0.056004296988248825,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8156249523162842,
+      "step": 1113
+    },
+    {
+      "completion_length": 117.84375,
+      "epoch": 1.4853333333333334,
+      "grad_norm": 1.4676050349808696,
+      "kl": 0.05859375,
+      "learning_rate": 2.5733333333333336e-07,
+      "loss": 0.0023,
+      "reward": 1.847916603088379,
+      "reward_std": 0.06361493468284607,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8479167222976685,
+      "step": 1114
+    },
+    {
+      "completion_length": 120.328125,
+      "epoch": 1.4866666666666668,
+      "grad_norm": 2.5149280183141602,
+      "kl": 0.07666015625,
+      "learning_rate": 2.5666666666666666e-07,
+      "loss": 0.0031,
+      "reward": 1.859635353088379,
+      "reward_std": 0.1264999657869339,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8596354722976685,
+      "step": 1115
+    },
+    {
+      "completion_length": 126.453125,
+      "epoch": 1.488,
+      "grad_norm": 1.069004678155317,
+      "kl": 0.0260009765625,
+      "learning_rate": 2.56e-07,
+      "loss": 0.001,
+      "reward": 1.806175708770752,
+      "reward_std": 0.048733197152614594,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8530505895614624,
+      "step": 1116
+    },
+    {
+      "completion_length": 120.5625,
+      "epoch": 1.4893333333333334,
+      "grad_norm": 0.12151057451745644,
+      "kl": 0.03369140625,
+      "learning_rate": 2.5533333333333337e-07,
+      "loss": 0.0013,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 1.0,
+      "step": 1117
+    },
+    {
+      "completion_length": 114.234375,
+      "epoch": 1.4906666666666666,
+      "grad_norm": 1.989133440847777,
+      "kl": 0.050048828125,
+      "learning_rate": 2.546666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.8432291746139526,
+      "reward_std": 0.11930285394191742,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8432291746139526,
+      "step": 1118
+    },
+    {
+      "completion_length": 123.46875,
+      "epoch": 1.492,
+      "grad_norm": 2.1774301040373394,
+      "kl": 0.049072265625,
+      "learning_rate": 2.5399999999999997e-07,
+      "loss": 0.002,
+      "reward": 1.8973958492279053,
+      "reward_std": 0.0981520563364029,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8973958492279053,
+      "step": 1119
+    },
+    {
+      "completion_length": 123.59375,
+      "epoch": 1.4933333333333334,
+      "grad_norm": 1.461587421927434,
+      "kl": 0.06298828125,
+      "learning_rate": 2.533333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.8832589387893677,
+      "reward_std": 0.062371283769607544,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8832589387893677,
+      "step": 1120
+    },
+    {
+      "completion_length": 121.375,
+      "epoch": 1.4946666666666666,
+      "grad_norm": 2.6331586058190073,
+      "kl": 0.044921875,
+      "learning_rate": 2.526666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.9166667461395264,
+      "reward_std": 0.1041666641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9166666865348816,
+      "step": 1121
+    },
+    {
+      "completion_length": 115.734375,
+      "epoch": 1.496,
+      "grad_norm": 1.2770023847402725,
+      "kl": 0.052734375,
+      "learning_rate": 2.52e-07,
+      "loss": 0.0021,
+      "reward": 1.9296875,
+      "reward_std": 0.08295939117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9296875,
+      "step": 1122
+    },
+    {
+      "completion_length": 120.296875,
+      "epoch": 1.4973333333333334,
+      "grad_norm": 20.920195380337816,
+      "kl": 0.05908203125,
+      "learning_rate": 2.5133333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.8004837036132812,
+      "reward_std": 0.06027135252952576,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8004835844039917,
+      "step": 1123
+    },
+    {
+      "completion_length": 111.90625,
+      "epoch": 1.4986666666666666,
+      "grad_norm": 2.4716170186835567,
+      "kl": 0.05126953125,
+      "learning_rate": 2.5066666666666663e-07,
+      "loss": 0.0021,
+      "reward": 1.8895833492279053,
+      "reward_std": 0.07424402981996536,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8895833492279053,
+      "step": 1124
+    },
+    {
+      "completion_length": 129.203125,
+      "epoch": 1.5,
+      "grad_norm": 1.2537174788886662,
+      "kl": 0.034423828125,
+      "learning_rate": 2.5e-07,
+      "loss": 0.0014,
+      "reward": 1.828125,
+      "reward_std": 0.11544691026210785,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.859375,
+      "step": 1125
+    },
+    {
+      "completion_length": 109.59375,
+      "epoch": 1.5013333333333332,
+      "grad_norm": 1.3666047974908067,
+      "kl": 0.03466796875,
+      "learning_rate": 2.493333333333333e-07,
+      "loss": 0.0014,
+      "reward": 1.877343773841858,
+      "reward_std": 0.02135416492819786,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8773437738418579,
+      "step": 1126
+    },
+    {
+      "completion_length": 124.046875,
+      "epoch": 1.5026666666666668,
+      "grad_norm": 0.9461998075034196,
+      "kl": 0.05078125,
+      "learning_rate": 2.4866666666666664e-07,
+      "loss": 0.002,
+      "reward": 1.868749976158142,
+      "reward_std": 0.07452812790870667,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8687499761581421,
+      "step": 1127
+    },
+    {
+      "completion_length": 125.109375,
+      "epoch": 1.504,
+      "grad_norm": 1.878368160662916,
+      "kl": 0.032470703125,
+      "learning_rate": 2.48e-07,
+      "loss": 0.0013,
+      "reward": 1.889062523841858,
+      "reward_std": 0.109375,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9046875238418579,
+      "step": 1128
+    },
+    {
+      "completion_length": 116.390625,
+      "epoch": 1.5053333333333332,
+      "grad_norm": 1.2510447652613044,
+      "kl": 0.04052734375,
+      "learning_rate": 2.473333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.8541667461395264,
+      "reward_std": 0.08537659049034119,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8541666865348816,
+      "step": 1129
+    },
+    {
+      "completion_length": 119.9375,
+      "epoch": 1.5066666666666668,
+      "grad_norm": 3.0391839626071526,
+      "kl": 0.078125,
+      "learning_rate": 2.4666666666666665e-07,
+      "loss": 0.0031,
+      "reward": 1.7694940567016602,
+      "reward_std": 0.1484019160270691,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7851190567016602,
+      "step": 1130
+    },
+    {
+      "completion_length": 113.53125,
+      "epoch": 1.508,
+      "grad_norm": 3.652259472542223,
+      "kl": 0.031494140625,
+      "learning_rate": 2.46e-07,
+      "loss": 0.0013,
+      "reward": 1.844010353088379,
+      "reward_std": 0.017800752073526382,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8440104722976685,
+      "step": 1131
+    },
+    {
+      "completion_length": 117.015625,
+      "epoch": 1.5093333333333332,
+      "grad_norm": 1.4876216462526528,
+      "kl": 0.036865234375,
+      "learning_rate": 2.453333333333333e-07,
+      "loss": 0.0015,
+      "reward": 1.9343750476837158,
+      "reward_std": 0.08348139375448227,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9343750476837158,
+      "step": 1132
+    },
+    {
+      "completion_length": 117.265625,
+      "epoch": 1.5106666666666668,
+      "grad_norm": 1.3247841305411205,
+      "kl": 0.042724609375,
+      "learning_rate": 2.4466666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.7760417461395264,
+      "reward_std": 0.08714609593153,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7760416865348816,
+      "step": 1133
+    },
+    {
+      "completion_length": 130.234375,
+      "epoch": 1.512,
+      "grad_norm": 1.5462057015736572,
+      "kl": 0.0203857421875,
+      "learning_rate": 2.4399999999999996e-07,
+      "loss": 0.0008,
+      "reward": 1.8346354961395264,
+      "reward_std": 0.21199540793895721,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8346354365348816,
+      "step": 1134
+    },
+    {
+      "completion_length": 130.375,
+      "epoch": 1.5133333333333332,
+      "grad_norm": 2.3076604606403364,
+      "kl": 0.07568359375,
+      "learning_rate": 2.433333333333333e-07,
+      "loss": 0.003,
+      "reward": 1.8604166507720947,
+      "reward_std": 0.20503023266792297,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8760416507720947,
+      "step": 1135
+    },
+    {
+      "completion_length": 132.21875,
+      "epoch": 1.5146666666666668,
+      "grad_norm": 2.103338909080985,
+      "kl": 0.04833984375,
+      "learning_rate": 2.4266666666666667e-07,
+      "loss": 0.0019,
+      "reward": 1.7979166507720947,
+      "reward_std": 0.16420939564704895,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8135417103767395,
+      "step": 1136
+    },
+    {
+      "completion_length": 121.890625,
+      "epoch": 1.516,
+      "grad_norm": 1.8098951274112574,
+      "kl": 0.0439453125,
+      "learning_rate": 2.4199999999999997e-07,
+      "loss": 0.0018,
+      "reward": 1.6947916746139526,
+      "reward_std": 0.11990776658058167,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6947916746139526,
+      "step": 1137
+    },
+    {
+      "completion_length": 122.171875,
+      "epoch": 1.5173333333333332,
+      "grad_norm": 1.1948116130181585,
+      "kl": 0.059326171875,
+      "learning_rate": 2.413333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.8072917461395264,
+      "reward_std": 0.04946783930063248,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8229166269302368,
+      "step": 1138
+    },
+    {
+      "completion_length": 112.25,
+      "epoch": 1.5186666666666668,
+      "grad_norm": 1.0652263477901731,
+      "kl": 0.044189453125,
+      "learning_rate": 2.406666666666667e-07,
+      "loss": 0.0018,
+      "reward": 1.9148437976837158,
+      "reward_std": 0.015490744262933731,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9148436784744263,
+      "step": 1139
+    },
+    {
+      "completion_length": 116.25,
+      "epoch": 1.52,
+      "grad_norm": 7.958427960689194,
+      "kl": 0.0498046875,
+      "learning_rate": 2.4e-07,
+      "loss": 0.002,
+      "reward": 1.9276041984558105,
+      "reward_std": 0.030058756470680237,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9276041984558105,
+      "step": 1140
+    },
+    {
+      "completion_length": 139.140625,
+      "epoch": 1.5213333333333332,
+      "grad_norm": 3.0769394228700406,
+      "kl": 0.0595703125,
+      "learning_rate": 2.3933333333333333e-07,
+      "loss": 0.0024,
+      "reward": 1.693750023841858,
+      "reward_std": 0.22085581719875336,
+      "rewards/format_reward": 0.90625,
+      "rewards/iou_reward": 0.7875000238418579,
+      "step": 1141
+    },
+    {
+      "completion_length": 129.734375,
+      "epoch": 1.5226666666666666,
+      "grad_norm": 1.7836268359308978,
+      "kl": 0.04541015625,
+      "learning_rate": 2.3866666666666663e-07,
+      "loss": 0.0018,
+      "reward": 1.745833396911621,
+      "reward_std": 0.13587498664855957,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7614582777023315,
+      "step": 1142
+    },
+    {
+      "completion_length": 121.640625,
+      "epoch": 1.524,
+      "grad_norm": 3.897129047662442,
+      "kl": 0.1875,
+      "learning_rate": 2.38e-07,
+      "loss": 0.0075,
+      "reward": 1.7994792461395264,
+      "reward_std": 0.14545938372612,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7994791269302368,
+      "step": 1143
+    },
+    {
+      "completion_length": 119.0,
+      "epoch": 1.5253333333333332,
+      "grad_norm": 1.4338026278720053,
+      "kl": 0.04150390625,
+      "learning_rate": 2.3733333333333334e-07,
+      "loss": 0.0017,
+      "reward": 1.8528646230697632,
+      "reward_std": 0.1081450954079628,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8528646230697632,
+      "step": 1144
+    },
+    {
+      "completion_length": 119.25,
+      "epoch": 1.5266666666666666,
+      "grad_norm": 2.037262363601687,
+      "kl": 0.045654296875,
+      "learning_rate": 2.3666666666666664e-07,
+      "loss": 0.0018,
+      "reward": 1.816145896911621,
+      "reward_std": 0.11770834028720856,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8161458373069763,
+      "step": 1145
+    },
+    {
+      "completion_length": 118.65625,
+      "epoch": 1.528,
+      "grad_norm": 1.2479078390137524,
+      "kl": 0.05810546875,
+      "learning_rate": 2.3599999999999997e-07,
+      "loss": 0.0023,
+      "reward": 1.8177083730697632,
+      "reward_std": 0.12394770234823227,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8333333730697632,
+      "step": 1146
+    },
+    {
+      "completion_length": 121.96875,
+      "epoch": 1.5293333333333332,
+      "grad_norm": 1.6444815067466316,
+      "kl": 0.041259765625,
+      "learning_rate": 2.3533333333333332e-07,
+      "loss": 0.0017,
+      "reward": 1.8583333492279053,
+      "reward_std": 0.08461841940879822,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8895832896232605,
+      "step": 1147
+    },
+    {
+      "completion_length": 117.046875,
+      "epoch": 1.5306666666666666,
+      "grad_norm": 1.2410917528092293,
+      "kl": 0.0595703125,
+      "learning_rate": 2.3466666666666665e-07,
+      "loss": 0.0024,
+      "reward": 1.9025298357009888,
+      "reward_std": 0.04352487251162529,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.902529776096344,
+      "step": 1148
+    },
+    {
+      "completion_length": 120.015625,
+      "epoch": 1.532,
+      "grad_norm": 1.0255357144891095,
+      "kl": 0.03564453125,
+      "learning_rate": 2.34e-07,
+      "loss": 0.0014,
+      "reward": 1.8385417461395264,
+      "reward_std": 0.1090010553598404,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8385416865348816,
+      "step": 1149
+    },
+    {
+      "completion_length": 127.8125,
+      "epoch": 1.5333333333333332,
+      "grad_norm": 1.1530380083604892,
+      "kl": 0.044189453125,
+      "learning_rate": 2.3333333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.740625023841858,
+      "reward_std": 0.09375,
+      "rewards/format_reward": 0.9375,
+      "rewards/iou_reward": 0.8031250238418579,
+      "step": 1150
+    },
+    {
+      "completion_length": 115.78125,
+      "epoch": 1.5346666666666666,
+      "grad_norm": 2.2598908665618356,
+      "kl": 0.0556640625,
+      "learning_rate": 2.3266666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.9466146230697632,
+      "reward_std": 0.0390625037252903,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9466146230697632,
+      "step": 1151
+    },
+    {
+      "completion_length": 116.203125,
+      "epoch": 1.536,
+      "grad_norm": 5.243733550240576,
+      "kl": 0.03662109375,
+      "learning_rate": 2.32e-07,
+      "loss": 0.0015,
+      "reward": 1.738020896911621,
+      "reward_std": 0.10071748495101929,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7380208373069763,
+      "step": 1152
+    },
+    {
+      "completion_length": 110.265625,
+      "epoch": 1.5373333333333332,
+      "grad_norm": 3.3198274664020313,
+      "kl": 0.060302734375,
+      "learning_rate": 2.3133333333333331e-07,
+      "loss": 0.0024,
+      "reward": 1.9424107074737549,
+      "reward_std": 0.06914547085762024,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9424107074737549,
+      "step": 1153
+    },
+    {
+      "completion_length": 119.59375,
+      "epoch": 1.5386666666666666,
+      "grad_norm": 1.3920585142327406,
+      "kl": 0.03271484375,
+      "learning_rate": 2.3066666666666664e-07,
+      "loss": 0.0013,
+      "reward": 1.90625,
+      "reward_std": 0.09248244762420654,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9062500596046448,
+      "step": 1154
+    },
+    {
+      "completion_length": 118.140625,
+      "epoch": 1.54,
+      "grad_norm": 2.036002037086608,
+      "kl": 0.031982421875,
+      "learning_rate": 2.3e-07,
+      "loss": 0.0013,
+      "reward": 1.939062476158142,
+      "reward_std": 0.08357236534357071,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9390624761581421,
+      "step": 1155
+    },
+    {
+      "completion_length": 130.4375,
+      "epoch": 1.5413333333333332,
+      "grad_norm": 19.147177075204702,
+      "kl": 0.05224609375,
+      "learning_rate": 2.2933333333333332e-07,
+      "loss": 0.0021,
+      "reward": 1.8452008962631226,
+      "reward_std": 0.1372063159942627,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8608258962631226,
+      "step": 1156
+    },
+    {
+      "completion_length": 127.296875,
+      "epoch": 1.5426666666666666,
+      "grad_norm": 2.148405582351816,
+      "kl": 0.06005859375,
+      "learning_rate": 2.2866666666666665e-07,
+      "loss": 0.0024,
+      "reward": 1.7510416507720947,
+      "reward_std": 0.1257309466600418,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7666666507720947,
+      "step": 1157
+    },
+    {
+      "completion_length": 132.453125,
+      "epoch": 1.544,
+      "grad_norm": 2.062126322872828,
+      "kl": 0.068359375,
+      "learning_rate": 2.28e-07,
+      "loss": 0.0027,
+      "reward": 1.6307291984558105,
+      "reward_std": 0.14925718307495117,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.6776041984558105,
+      "step": 1158
+    },
+    {
+      "completion_length": 123.96875,
+      "epoch": 1.5453333333333332,
+      "grad_norm": 1.4448528775401128,
+      "kl": 0.05517578125,
+      "learning_rate": 2.2733333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.781510353088379,
+      "reward_std": 0.06942923367023468,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7971354722976685,
+      "step": 1159
+    },
+    {
+      "completion_length": 129.671875,
+      "epoch": 1.5466666666666666,
+      "grad_norm": 2.520803936469217,
+      "kl": 0.054443359375,
+      "learning_rate": 2.2666666666666663e-07,
+      "loss": 0.0022,
+      "reward": 1.7263020277023315,
+      "reward_std": 0.19206523895263672,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7263020873069763,
+      "step": 1160
+    },
+    {
+      "completion_length": 126.90625,
+      "epoch": 1.548,
+      "grad_norm": 2.6553361069880608,
+      "kl": 0.061279296875,
+      "learning_rate": 2.2599999999999999e-07,
+      "loss": 0.0024,
+      "reward": 1.8565104007720947,
+      "reward_std": 0.13303713500499725,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8565104603767395,
+      "step": 1161
+    },
+    {
+      "completion_length": 119.953125,
+      "epoch": 1.5493333333333332,
+      "grad_norm": 1.4492424072991414,
+      "kl": 0.058349609375,
+      "learning_rate": 2.253333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.7851449251174927,
+      "reward_std": 0.06905744969844818,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8007699251174927,
+      "step": 1162
+    },
+    {
+      "completion_length": 123.09375,
+      "epoch": 1.5506666666666666,
+      "grad_norm": 1.1065404668616003,
+      "kl": 0.0517578125,
+      "learning_rate": 2.2466666666666664e-07,
+      "loss": 0.0021,
+      "reward": 1.7213542461395264,
+      "reward_std": 0.09085840731859207,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7369792461395264,
+      "step": 1163
+    },
+    {
+      "completion_length": 127.140625,
+      "epoch": 1.552,
+      "grad_norm": 2.027193008834877,
+      "kl": 0.051025390625,
+      "learning_rate": 2.24e-07,
+      "loss": 0.002,
+      "reward": 1.8188802003860474,
+      "reward_std": 0.10411254316568375,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8345053195953369,
+      "step": 1164
+    },
+    {
+      "completion_length": 117.828125,
+      "epoch": 1.5533333333333332,
+      "grad_norm": 3.6383905712974896,
+      "kl": 0.046875,
+      "learning_rate": 2.2333333333333332e-07,
+      "loss": 0.0019,
+      "reward": 1.7548363208770752,
+      "reward_std": 0.17082113027572632,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7704612612724304,
+      "step": 1165
+    },
+    {
+      "completion_length": 125.375,
+      "epoch": 1.5546666666666666,
+      "grad_norm": 2.8783060183547176,
+      "kl": 0.06982421875,
+      "learning_rate": 2.2266666666666668e-07,
+      "loss": 0.0028,
+      "reward": 1.8648995161056519,
+      "reward_std": 0.09027612954378128,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8648995161056519,
+      "step": 1166
+    },
+    {
+      "completion_length": 112.5625,
+      "epoch": 1.556,
+      "grad_norm": 1.7847856774248632,
+      "kl": 0.07080078125,
+      "learning_rate": 2.22e-07,
+      "loss": 0.0028,
+      "reward": 1.726822853088379,
+      "reward_std": 0.1127113625407219,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7268228530883789,
+      "step": 1167
+    },
+    {
+      "completion_length": 121.375,
+      "epoch": 1.5573333333333332,
+      "grad_norm": 3.338526693474592,
+      "kl": 0.04248046875,
+      "learning_rate": 2.213333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.765625,
+      "reward_std": 0.1447872668504715,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7656249403953552,
+      "step": 1168
+    },
+    {
+      "completion_length": 117.625,
+      "epoch": 1.5586666666666666,
+      "grad_norm": 1.1508093208606325,
+      "kl": 0.03955078125,
+      "learning_rate": 2.2066666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.8182291984558105,
+      "reward_std": 0.12754753232002258,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8338541984558105,
+      "step": 1169
+    },
+    {
+      "completion_length": 122.203125,
+      "epoch": 1.56,
+      "grad_norm": 1.6565122078362131,
+      "kl": 0.0390625,
+      "learning_rate": 2.1999999999999998e-07,
+      "loss": 0.0016,
+      "reward": 1.921875,
+      "reward_std": 0.10577812790870667,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.921875,
+      "step": 1170
+    },
+    {
+      "completion_length": 126.078125,
+      "epoch": 1.5613333333333332,
+      "grad_norm": 1.5446027970254128,
+      "kl": 0.04443359375,
+      "learning_rate": 2.193333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.7919270992279053,
+      "reward_std": 0.11630132049322128,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8075520992279053,
+      "step": 1171
+    },
+    {
+      "completion_length": 114.015625,
+      "epoch": 1.5626666666666666,
+      "grad_norm": 1.0869377926781711,
+      "kl": 0.0439453125,
+      "learning_rate": 2.1866666666666667e-07,
+      "loss": 0.0018,
+      "reward": 1.9388021230697632,
+      "reward_std": 0.0182291641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9388021230697632,
+      "step": 1172
+    },
+    {
+      "completion_length": 109.375,
+      "epoch": 1.564,
+      "grad_norm": 3.1467163342927473,
+      "kl": 0.049560546875,
+      "learning_rate": 2.18e-07,
+      "loss": 0.002,
+      "reward": 1.8776042461395264,
+      "reward_std": 0.11904378235340118,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8776041269302368,
+      "step": 1173
+    },
+    {
+      "completion_length": 120.734375,
+      "epoch": 1.5653333333333332,
+      "grad_norm": 1.2320729468012683,
+      "kl": 0.039306640625,
+      "learning_rate": 2.1733333333333332e-07,
+      "loss": 0.0016,
+      "reward": 1.8493304252624512,
+      "reward_std": 0.08284883201122284,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8493303656578064,
+      "step": 1174
+    },
+    {
+      "completion_length": 121.703125,
+      "epoch": 1.5666666666666667,
+      "grad_norm": 1.5785776896945538,
+      "kl": 0.048828125,
+      "learning_rate": 2.1666666666666667e-07,
+      "loss": 0.0019,
+      "reward": 1.8810639381408691,
+      "reward_std": 0.09754080325365067,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8810639977455139,
+      "step": 1175
+    },
+    {
+      "completion_length": 120.40625,
+      "epoch": 1.568,
+      "grad_norm": 1.030161416914008,
+      "kl": 0.0311279296875,
+      "learning_rate": 2.1599999999999998e-07,
+      "loss": 0.0012,
+      "reward": 1.8953125476837158,
+      "reward_std": 0.04443775862455368,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.895312488079071,
+      "step": 1176
+    },
+    {
+      "completion_length": 114.578125,
+      "epoch": 1.5693333333333332,
+      "grad_norm": 1.188795331611464,
+      "kl": 0.041015625,
+      "learning_rate": 2.153333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.8802083730697632,
+      "reward_std": 0.05639805644750595,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8802083730697632,
+      "step": 1177
+    },
+    {
+      "completion_length": 139.046875,
+      "epoch": 1.5706666666666667,
+      "grad_norm": 3.1641218954871926,
+      "kl": 0.049072265625,
+      "learning_rate": 2.1466666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.663802146911621,
+      "reward_std": 0.18176907300949097,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.7106771469116211,
+      "step": 1178
+    },
+    {
+      "completion_length": 110.40625,
+      "epoch": 1.572,
+      "grad_norm": 4.602850769733478,
+      "kl": 0.037841796875,
+      "learning_rate": 2.1399999999999998e-07,
+      "loss": 0.0015,
+      "reward": 1.9114583730697632,
+      "reward_std": 0.04847204312682152,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9114583134651184,
+      "step": 1179
+    },
+    {
+      "completion_length": 115.390625,
+      "epoch": 1.5733333333333333,
+      "grad_norm": 1.8255266094640448,
+      "kl": 0.06591796875,
+      "learning_rate": 2.1333333333333334e-07,
+      "loss": 0.0026,
+      "reward": 1.7161458730697632,
+      "reward_std": 0.18255561590194702,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.7473958134651184,
+      "step": 1180
+    },
+    {
+      "completion_length": 124.0625,
+      "epoch": 1.5746666666666667,
+      "grad_norm": 1.9655475142553216,
+      "kl": 0.07177734375,
+      "learning_rate": 2.1266666666666667e-07,
+      "loss": 0.0029,
+      "reward": 1.8363466262817383,
+      "reward_std": 0.12291628122329712,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8519717454910278,
+      "step": 1181
+    },
+    {
+      "completion_length": 116.5625,
+      "epoch": 1.576,
+      "grad_norm": 3.646538915884861,
+      "kl": 0.05078125,
+      "learning_rate": 2.12e-07,
+      "loss": 0.002,
+      "reward": 1.76953125,
+      "reward_std": 0.07907126098871231,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.76953125,
+      "step": 1182
+    },
+    {
+      "completion_length": 128.09375,
+      "epoch": 1.5773333333333333,
+      "grad_norm": 1.1179553084795544,
+      "kl": 0.04638671875,
+      "learning_rate": 2.1133333333333335e-07,
+      "loss": 0.0019,
+      "reward": 1.78125,
+      "reward_std": 0.11839609593153,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.796875,
+      "step": 1183
+    },
+    {
+      "completion_length": 124.203125,
+      "epoch": 1.5786666666666667,
+      "grad_norm": 1.3242454665901164,
+      "kl": 0.052490234375,
+      "learning_rate": 2.1066666666666665e-07,
+      "loss": 0.0021,
+      "reward": 1.8015625476837158,
+      "reward_std": 0.11203107237815857,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8015624284744263,
+      "step": 1184
+    },
+    {
+      "completion_length": 125.375,
+      "epoch": 1.58,
+      "grad_norm": 2.2789241903302444,
+      "kl": 0.078125,
+      "learning_rate": 2.0999999999999997e-07,
+      "loss": 0.0031,
+      "reward": 1.7844493389129639,
+      "reward_std": 0.1811625361442566,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8313243985176086,
+      "step": 1185
+    },
+    {
+      "completion_length": 122.890625,
+      "epoch": 1.5813333333333333,
+      "grad_norm": 4.414815674713166,
+      "kl": 0.040771484375,
+      "learning_rate": 2.0933333333333333e-07,
+      "loss": 0.0016,
+      "reward": 1.7708333730697632,
+      "reward_std": 0.2181294858455658,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8020833730697632,
+      "step": 1186
+    },
+    {
+      "completion_length": 125.578125,
+      "epoch": 1.5826666666666667,
+      "grad_norm": 1.0688330388325966,
+      "kl": 0.0245361328125,
+      "learning_rate": 2.0866666666666666e-07,
+      "loss": 0.001,
+      "reward": 1.9106770753860474,
+      "reward_std": 0.06718749552965164,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9263020753860474,
+      "step": 1187
+    },
+    {
+      "completion_length": 121.359375,
+      "epoch": 1.584,
+      "grad_norm": 0.7399857568734487,
+      "kl": 0.03857421875,
+      "learning_rate": 2.0799999999999998e-07,
+      "loss": 0.0015,
+      "reward": 1.7239583730697632,
+      "reward_std": 0.0625,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7395833730697632,
+      "step": 1188
+    },
+    {
+      "completion_length": 149.296875,
+      "epoch": 1.5853333333333333,
+      "grad_norm": 3.9585708721394184,
+      "kl": 0.060302734375,
+      "learning_rate": 2.0733333333333334e-07,
+      "loss": 0.0024,
+      "reward": 1.6091517210006714,
+      "reward_std": 0.17028123140335083,
+      "rewards/format_reward": 0.921875,
+      "rewards/iou_reward": 0.6872767210006714,
+      "step": 1189
+    },
+    {
+      "completion_length": 119.109375,
+      "epoch": 1.5866666666666667,
+      "grad_norm": 1.7807782457211383,
+      "kl": 0.04345703125,
+      "learning_rate": 2.0666666666666666e-07,
+      "loss": 0.0017,
+      "reward": 1.7697917222976685,
+      "reward_std": 0.09907464683055878,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7697916626930237,
+      "step": 1190
+    },
+    {
+      "completion_length": 118.171875,
+      "epoch": 1.588,
+      "grad_norm": 2.0271657240389724,
+      "kl": 0.0546875,
+      "learning_rate": 2.06e-07,
+      "loss": 0.0022,
+      "reward": 1.8093750476837158,
+      "reward_std": 0.09929218888282776,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.840624988079071,
+      "step": 1191
+    },
+    {
+      "completion_length": 123.171875,
+      "epoch": 1.5893333333333333,
+      "grad_norm": 1.4870553323850688,
+      "kl": 0.033935546875,
+      "learning_rate": 2.0533333333333332e-07,
+      "loss": 0.0014,
+      "reward": 1.9336681365966797,
+      "reward_std": 0.08769624680280685,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9336681962013245,
+      "step": 1192
+    },
+    {
+      "completion_length": 125.65625,
+      "epoch": 1.5906666666666667,
+      "grad_norm": 1.3378427911269497,
+      "kl": 0.0517578125,
+      "learning_rate": 2.0466666666666665e-07,
+      "loss": 0.0021,
+      "reward": 1.8098958730697632,
+      "reward_std": 0.10653018951416016,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8411458730697632,
+      "step": 1193
+    },
+    {
+      "completion_length": 111.625,
+      "epoch": 1.592,
+      "grad_norm": 2.954689208840636,
+      "kl": 0.025390625,
+      "learning_rate": 2.0399999999999997e-07,
+      "loss": 0.001,
+      "reward": 1.7760417461395264,
+      "reward_std": 0.2075854390859604,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7760416865348816,
+      "step": 1194
+    },
+    {
+      "completion_length": 111.71875,
+      "epoch": 1.5933333333333333,
+      "grad_norm": 1.1530612838208638,
+      "kl": 0.0537109375,
+      "learning_rate": 2.0333333333333333e-07,
+      "loss": 0.0021,
+      "reward": 1.9229910373687744,
+      "reward_std": 0.027759974822402,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9229910969734192,
+      "step": 1195
+    },
+    {
+      "completion_length": 129.828125,
+      "epoch": 1.5946666666666667,
+      "grad_norm": 3.876297905393554,
+      "kl": 0.05078125,
+      "learning_rate": 2.0266666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.8341145515441895,
+      "reward_std": 0.14217311143875122,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8341146111488342,
+      "step": 1196
+    },
+    {
+      "completion_length": 128.96875,
+      "epoch": 1.596,
+      "grad_norm": 1.7469614613106716,
+      "kl": 0.032958984375,
+      "learning_rate": 2.02e-07,
+      "loss": 0.0013,
+      "reward": 1.761458396911621,
+      "reward_std": 0.1854945570230484,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7770833373069763,
+      "step": 1197
+    },
+    {
+      "completion_length": 123.046875,
+      "epoch": 1.5973333333333333,
+      "grad_norm": 2.039523630502532,
+      "kl": 0.0419921875,
+      "learning_rate": 2.0133333333333334e-07,
+      "loss": 0.0017,
+      "reward": 1.806249976158142,
+      "reward_std": 0.12083333730697632,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8062499761581421,
+      "step": 1198
+    },
+    {
+      "completion_length": 124.015625,
+      "epoch": 1.5986666666666667,
+      "grad_norm": 0.9318209744860997,
+      "kl": 0.0556640625,
+      "learning_rate": 2.0066666666666666e-07,
+      "loss": 0.0022,
+      "reward": 1.8450521230697632,
+      "reward_std": 0.0442708358168602,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8450520038604736,
+      "step": 1199
+    },
+    {
+      "completion_length": 121.140625,
+      "epoch": 1.6,
+      "grad_norm": 5.238344655458376,
+      "kl": 0.0712890625,
+      "learning_rate": 2e-07,
+      "loss": 0.0028,
+      "reward": 1.8846354484558105,
+      "reward_std": 0.08126640319824219,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8846354484558105,
+      "step": 1200
+    },
+    {
+      "completion_length": 115.234375,
+      "epoch": 1.6013333333333333,
+      "grad_norm": 5.2273204616485796,
+      "kl": 0.078125,
+      "learning_rate": 1.9933333333333332e-07,
+      "loss": 0.0031,
+      "reward": 1.8820312023162842,
+      "reward_std": 0.07820618897676468,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8820312023162842,
+      "step": 1201
+    },
+    {
+      "completion_length": 119.390625,
+      "epoch": 1.6026666666666667,
+      "grad_norm": 1.8234911960258817,
+      "kl": 0.05078125,
+      "learning_rate": 1.9866666666666665e-07,
+      "loss": 0.002,
+      "reward": 1.7916667461395264,
+      "reward_std": 0.15786145627498627,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8072917461395264,
+      "step": 1202
+    },
+    {
+      "completion_length": 124.4375,
+      "epoch": 1.604,
+      "grad_norm": 3.7459686495333595,
+      "kl": 0.04833984375,
+      "learning_rate": 1.98e-07,
+      "loss": 0.0019,
+      "reward": 1.813020944595337,
+      "reward_std": 0.1614583283662796,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8286458253860474,
+      "step": 1203
+    },
+    {
+      "completion_length": 118.609375,
+      "epoch": 1.6053333333333333,
+      "grad_norm": 2.2023362106827036,
+      "kl": 0.054931640625,
+      "learning_rate": 1.9733333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.8796131610870361,
+      "reward_std": 0.15003132820129395,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8796131014823914,
+      "step": 1204
+    },
+    {
+      "completion_length": 112.328125,
+      "epoch": 1.6066666666666667,
+      "grad_norm": 2.72775013481147,
+      "kl": 0.048095703125,
+      "learning_rate": 1.9666666666666665e-07,
+      "loss": 0.0019,
+      "reward": 1.8526041507720947,
+      "reward_std": 0.19501066207885742,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8682291507720947,
+      "step": 1205
+    },
+    {
+      "completion_length": 116.515625,
+      "epoch": 1.608,
+      "grad_norm": 1.879389143014372,
+      "kl": 0.06982421875,
+      "learning_rate": 1.96e-07,
+      "loss": 0.0028,
+      "reward": 1.8284598588943481,
+      "reward_std": 0.07343515008687973,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8284597396850586,
+      "step": 1206
+    },
+    {
+      "completion_length": 120.71875,
+      "epoch": 1.6093333333333333,
+      "grad_norm": 4.892076551618888,
+      "kl": 0.06298828125,
+      "learning_rate": 1.953333333333333e-07,
+      "loss": 0.0025,
+      "reward": 1.8612537384033203,
+      "reward_std": 0.11361106485128403,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8768787384033203,
+      "step": 1207
+    },
+    {
+      "completion_length": 114.4375,
+      "epoch": 1.6106666666666667,
+      "grad_norm": 1.4384503770739998,
+      "kl": 0.0439453125,
+      "learning_rate": 1.9466666666666664e-07,
+      "loss": 0.0018,
+      "reward": 1.8583333492279053,
+      "reward_std": 0.05624990910291672,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8583332300186157,
+      "step": 1208
+    },
+    {
+      "completion_length": 106.65625,
+      "epoch": 1.612,
+      "grad_norm": 2.37616067180752,
+      "kl": 0.0625,
+      "learning_rate": 1.94e-07,
+      "loss": 0.0025,
+      "reward": 1.8536458015441895,
+      "reward_std": 0.1263088881969452,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.853645920753479,
+      "step": 1209
+    },
+    {
+      "completion_length": 121.484375,
+      "epoch": 1.6133333333333333,
+      "grad_norm": 14.170632142345301,
+      "kl": 0.049072265625,
+      "learning_rate": 1.9333333333333332e-07,
+      "loss": 0.002,
+      "reward": 1.816145896911621,
+      "reward_std": 0.027876410633325577,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8161458969116211,
+      "step": 1210
+    },
+    {
+      "completion_length": 114.984375,
+      "epoch": 1.6146666666666667,
+      "grad_norm": 1.9211184885349253,
+      "kl": 0.045166015625,
+      "learning_rate": 1.9266666666666667e-07,
+      "loss": 0.0018,
+      "reward": 1.8057291507720947,
+      "reward_std": 0.11354165524244308,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8057291507720947,
+      "step": 1211
+    },
+    {
+      "completion_length": 134.109375,
+      "epoch": 1.616,
+      "grad_norm": 1.360616926576774,
+      "kl": 0.052978515625,
+      "learning_rate": 1.92e-07,
+      "loss": 0.0021,
+      "reward": 1.9356770515441895,
+      "reward_std": 0.06056355684995651,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9356771111488342,
+      "step": 1212
+    },
+    {
+      "completion_length": 124.671875,
+      "epoch": 1.6173333333333333,
+      "grad_norm": 1.6041192627149614,
+      "kl": 0.042236328125,
+      "learning_rate": 1.9133333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.9330356121063232,
+      "reward_std": 0.047382742166519165,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9330357313156128,
+      "step": 1213
+    },
+    {
+      "completion_length": 128.5,
+      "epoch": 1.6186666666666667,
+      "grad_norm": 1.5314228229310047,
+      "kl": 0.031982421875,
+      "learning_rate": 1.9066666666666668e-07,
+      "loss": 0.0013,
+      "reward": 1.8515625,
+      "reward_std": 0.1302083432674408,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8515625,
+      "step": 1214
+    },
+    {
+      "completion_length": 119.40625,
+      "epoch": 1.62,
+      "grad_norm": 1.7297167050907616,
+      "kl": 0.0341796875,
+      "learning_rate": 1.8999999999999998e-07,
+      "loss": 0.0014,
+      "reward": 1.8447916507720947,
+      "reward_std": 0.08958333730697632,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8604166507720947,
+      "step": 1215
+    },
+    {
+      "completion_length": 121.296875,
+      "epoch": 1.6213333333333333,
+      "grad_norm": 1.0527620763058594,
+      "kl": 0.050048828125,
+      "learning_rate": 1.893333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.8309895992279053,
+      "reward_std": 0.03593749925494194,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8309895992279053,
+      "step": 1216
+    },
+    {
+      "completion_length": 115.703125,
+      "epoch": 1.6226666666666667,
+      "grad_norm": 1.9680259477386763,
+      "kl": 0.06884765625,
+      "learning_rate": 1.8866666666666666e-07,
+      "loss": 0.0027,
+      "reward": 1.823958396911621,
+      "reward_std": 0.1568215787410736,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8395833969116211,
+      "step": 1217
+    },
+    {
+      "completion_length": 120.6875,
+      "epoch": 1.624,
+      "grad_norm": 4.32307561350951,
+      "kl": 0.06787109375,
+      "learning_rate": 1.88e-07,
+      "loss": 0.0027,
+      "reward": 1.8572545051574707,
+      "reward_std": 0.1421475112438202,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8728794455528259,
+      "step": 1218
+    },
+    {
+      "completion_length": 125.953125,
+      "epoch": 1.6253333333333333,
+      "grad_norm": 2.410963837870934,
+      "kl": 0.06298828125,
+      "learning_rate": 1.8733333333333332e-07,
+      "loss": 0.0025,
+      "reward": 1.802343726158142,
+      "reward_std": 0.16437631845474243,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8179687261581421,
+      "step": 1219
+    },
+    {
+      "completion_length": 120.59375,
+      "epoch": 1.6266666666666667,
+      "grad_norm": 7.07089606050777,
+      "kl": 0.033447265625,
+      "learning_rate": 1.8666666666666667e-07,
+      "loss": 0.0013,
+      "reward": 1.9236979484558105,
+      "reward_std": 0.09493855386972427,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9236979484558105,
+      "step": 1220
+    },
+    {
+      "completion_length": 124.1875,
+      "epoch": 1.6280000000000001,
+      "grad_norm": 9.355151340174691,
+      "kl": 0.0712890625,
+      "learning_rate": 1.86e-07,
+      "loss": 0.0028,
+      "reward": 1.8697917461395264,
+      "reward_std": 0.06088123098015785,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8697916269302368,
+      "step": 1221
+    },
+    {
+      "completion_length": 144.46875,
+      "epoch": 1.6293333333333333,
+      "grad_norm": 1.1399451134167782,
+      "kl": 0.043701171875,
+      "learning_rate": 1.8533333333333333e-07,
+      "loss": 0.0018,
+      "reward": 1.8790922164916992,
+      "reward_std": 0.15605472028255463,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.925967276096344,
+      "step": 1222
+    },
+    {
+      "completion_length": 114.875,
+      "epoch": 1.6306666666666667,
+      "grad_norm": 1.581205793960358,
+      "kl": 0.033447265625,
+      "learning_rate": 1.8466666666666665e-07,
+      "loss": 0.0013,
+      "reward": 1.9192708730697632,
+      "reward_std": 0.1037927195429802,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9192707538604736,
+      "step": 1223
+    },
+    {
+      "completion_length": 112.8125,
+      "epoch": 1.6320000000000001,
+      "grad_norm": 3.7829493971130916,
+      "kl": 0.055908203125,
+      "learning_rate": 1.8399999999999998e-07,
+      "loss": 0.0022,
+      "reward": 1.8799479007720947,
+      "reward_std": 0.09784639626741409,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8799479007720947,
+      "step": 1224
+    },
+    {
+      "completion_length": 112.71875,
+      "epoch": 1.6333333333333333,
+      "grad_norm": 1.7578499531904779,
+      "kl": 0.04931640625,
+      "learning_rate": 1.833333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.788802146911621,
+      "reward_std": 0.07401283085346222,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7888020873069763,
+      "step": 1225
+    },
+    {
+      "completion_length": 123.34375,
+      "epoch": 1.6346666666666667,
+      "grad_norm": 1.0973244009892953,
+      "kl": 0.045654296875,
+      "learning_rate": 1.8266666666666666e-07,
+      "loss": 0.0018,
+      "reward": 1.9205729961395264,
+      "reward_std": 0.020237715914845467,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9205729365348816,
+      "step": 1226
+    },
+    {
+      "completion_length": 135.90625,
+      "epoch": 1.6360000000000001,
+      "grad_norm": 2.7075739544306088,
+      "kl": 0.057861328125,
+      "learning_rate": 1.82e-07,
+      "loss": 0.0023,
+      "reward": 1.7817708253860474,
+      "reward_std": 0.14072880148887634,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.8286458849906921,
+      "step": 1227
+    },
+    {
+      "completion_length": 108.890625,
+      "epoch": 1.6373333333333333,
+      "grad_norm": 6.847978928897055,
+      "kl": 0.061279296875,
+      "learning_rate": 1.8133333333333334e-07,
+      "loss": 0.0024,
+      "reward": 1.8901041746139526,
+      "reward_std": 0.12246610224246979,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9057291746139526,
+      "step": 1228
+    },
+    {
+      "completion_length": 129.75,
+      "epoch": 1.6386666666666667,
+      "grad_norm": 2.106255147593734,
+      "kl": 0.068359375,
+      "learning_rate": 1.8066666666666667e-07,
+      "loss": 0.0027,
+      "reward": 1.755133867263794,
+      "reward_std": 0.09959057718515396,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7707589864730835,
+      "step": 1229
+    },
+    {
+      "completion_length": 122.1875,
+      "epoch": 1.6400000000000001,
+      "grad_norm": 1.4165468096061145,
+      "kl": 0.039794921875,
+      "learning_rate": 1.8e-07,
+      "loss": 0.0016,
+      "reward": 1.8671875,
+      "reward_std": 0.15335839986801147,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8671875,
+      "step": 1230
+    },
+    {
+      "completion_length": 130.265625,
+      "epoch": 1.6413333333333333,
+      "grad_norm": 1.5236362080228247,
+      "kl": 0.0576171875,
+      "learning_rate": 1.7933333333333332e-07,
+      "loss": 0.0023,
+      "reward": 1.8255207538604736,
+      "reward_std": 0.14105679094791412,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8411458730697632,
+      "step": 1231
+    },
+    {
+      "completion_length": 116.609375,
+      "epoch": 1.6426666666666667,
+      "grad_norm": 2.288724933882334,
+      "kl": 0.053466796875,
+      "learning_rate": 1.7866666666666665e-07,
+      "loss": 0.0021,
+      "reward": 1.8361979722976685,
+      "reward_std": 0.07693612575531006,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8361979722976685,
+      "step": 1232
+    },
+    {
+      "completion_length": 108.796875,
+      "epoch": 1.6440000000000001,
+      "grad_norm": 0.9116665049602876,
+      "kl": 0.038330078125,
+      "learning_rate": 1.7799999999999998e-07,
+      "loss": 0.0015,
+      "reward": 1.912500023841858,
+      "reward_std": 0.03750000149011612,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9124999642372131,
+      "step": 1233
+    },
+    {
+      "completion_length": 114.390625,
+      "epoch": 1.6453333333333333,
+      "grad_norm": 1.1571010906519006,
+      "kl": 0.031005859375,
+      "learning_rate": 1.7733333333333333e-07,
+      "loss": 0.0012,
+      "reward": 1.847916603088379,
+      "reward_std": 0.06760311126708984,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8479166626930237,
+      "step": 1234
+    },
+    {
+      "completion_length": 122.78125,
+      "epoch": 1.6466666666666665,
+      "grad_norm": 1.692885260810997,
+      "kl": 0.025390625,
+      "learning_rate": 1.7666666666666666e-07,
+      "loss": 0.001,
+      "reward": 1.7578125,
+      "reward_std": 0.0989583283662796,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7578125,
+      "step": 1235
+    },
+    {
+      "completion_length": 113.96875,
+      "epoch": 1.6480000000000001,
+      "grad_norm": 2.082874478963856,
+      "kl": 0.051513671875,
+      "learning_rate": 1.76e-07,
+      "loss": 0.0021,
+      "reward": 1.882552146911621,
+      "reward_std": 0.09369566291570663,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8825520873069763,
+      "step": 1236
+    },
+    {
+      "completion_length": 115.953125,
+      "epoch": 1.6493333333333333,
+      "grad_norm": 5.0445358180384465,
+      "kl": 0.0400390625,
+      "learning_rate": 1.7533333333333334e-07,
+      "loss": 0.0016,
+      "reward": 1.6369792222976685,
+      "reward_std": 0.08920939266681671,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6369792222976685,
+      "step": 1237
+    },
+    {
+      "completion_length": 110.75,
+      "epoch": 1.6506666666666665,
+      "grad_norm": 1.4435290210115193,
+      "kl": 0.04345703125,
+      "learning_rate": 1.7466666666666667e-07,
+      "loss": 0.0017,
+      "reward": 1.8387277126312256,
+      "reward_std": 0.1398790329694748,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8387277126312256,
+      "step": 1238
+    },
+    {
+      "completion_length": 132.078125,
+      "epoch": 1.6520000000000001,
+      "grad_norm": 1.9306143558979736,
+      "kl": 0.05615234375,
+      "learning_rate": 1.7399999999999997e-07,
+      "loss": 0.0022,
+      "reward": 1.7177083492279053,
+      "reward_std": 0.12180020660161972,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7333333492279053,
+      "step": 1239
+    },
+    {
+      "completion_length": 120.984375,
+      "epoch": 1.6533333333333333,
+      "grad_norm": 1.0189530419940933,
+      "kl": 0.061767578125,
+      "learning_rate": 1.7333333333333332e-07,
+      "loss": 0.0025,
+      "reward": 1.8377604484558105,
+      "reward_std": 0.03868855908513069,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8377604484558105,
+      "step": 1240
+    },
+    {
+      "completion_length": 119.90625,
+      "epoch": 1.6546666666666665,
+      "grad_norm": 16.487875755290275,
+      "kl": 0.5390625,
+      "learning_rate": 1.7266666666666665e-07,
+      "loss": 0.0216,
+      "reward": 1.7552083730697632,
+      "reward_std": 0.0826711431145668,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7552083730697632,
+      "step": 1241
+    },
+    {
+      "completion_length": 117.890625,
+      "epoch": 1.6560000000000001,
+      "grad_norm": 1.2750209197835283,
+      "kl": 0.039794921875,
+      "learning_rate": 1.7199999999999998e-07,
+      "loss": 0.0016,
+      "reward": 1.886979103088379,
+      "reward_std": 0.04498439282178879,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8869791626930237,
+      "step": 1242
+    },
+    {
+      "completion_length": 111.1875,
+      "epoch": 1.6573333333333333,
+      "grad_norm": 1.1869738428084007,
+      "kl": 0.050048828125,
+      "learning_rate": 1.7133333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.7916667461395264,
+      "reward_std": 0.09375,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7916666865348816,
+      "step": 1243
+    },
+    {
+      "completion_length": 121.390625,
+      "epoch": 1.6586666666666665,
+      "grad_norm": 1.5132151275029377,
+      "kl": 0.047607421875,
+      "learning_rate": 1.7066666666666666e-07,
+      "loss": 0.0019,
+      "reward": 1.72265625,
+      "reward_std": 0.039632298052310944,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.72265625,
+      "step": 1244
+    },
+    {
+      "completion_length": 120.375,
+      "epoch": 1.6600000000000001,
+      "grad_norm": 1.5091942101756934,
+      "kl": 0.0517578125,
+      "learning_rate": 1.7000000000000001e-07,
+      "loss": 0.0021,
+      "reward": 1.876562476158142,
+      "reward_std": 0.10059293359518051,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8765624761581421,
+      "step": 1245
+    },
+    {
+      "completion_length": 117.90625,
+      "epoch": 1.6613333333333333,
+      "grad_norm": 1.457102530602572,
+      "kl": 0.042236328125,
+      "learning_rate": 1.6933333333333334e-07,
+      "loss": 0.0017,
+      "reward": 1.7453124523162842,
+      "reward_std": 0.11837605386972427,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.745312511920929,
+      "step": 1246
+    },
+    {
+      "completion_length": 117.515625,
+      "epoch": 1.6626666666666665,
+      "grad_norm": 2.5550454422832134,
+      "kl": 0.0634765625,
+      "learning_rate": 1.6866666666666664e-07,
+      "loss": 0.0025,
+      "reward": 1.8585937023162842,
+      "reward_std": 0.11685013771057129,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.858593761920929,
+      "step": 1247
+    },
+    {
+      "completion_length": 114.1875,
+      "epoch": 1.6640000000000001,
+      "grad_norm": 3.0424254355923157,
+      "kl": 0.055419921875,
+      "learning_rate": 1.68e-07,
+      "loss": 0.0022,
+      "reward": 1.9557292461395264,
+      "reward_std": 0.038069792091846466,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9557291269302368,
+      "step": 1248
+    },
+    {
+      "completion_length": 116.4375,
+      "epoch": 1.6653333333333333,
+      "grad_norm": 1.5619896928711547,
+      "kl": 0.039306640625,
+      "learning_rate": 1.6733333333333332e-07,
+      "loss": 0.0016,
+      "reward": 1.9427083730697632,
+      "reward_std": 0.06484109163284302,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9427083134651184,
+      "step": 1249
+    },
+    {
+      "completion_length": 109.453125,
+      "epoch": 1.6666666666666665,
+      "grad_norm": 1.8546584690946601,
+      "kl": 0.06787109375,
+      "learning_rate": 1.6666666666666665e-07,
+      "loss": 0.0027,
+      "reward": 1.8026041984558105,
+      "reward_std": 0.05937499552965164,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8026042580604553,
+      "step": 1250
+    },
+    {
+      "completion_length": 122.03125,
+      "epoch": 1.6680000000000001,
+      "grad_norm": 1.7252785794694487,
+      "kl": 0.05078125,
+      "learning_rate": 1.66e-07,
+      "loss": 0.002,
+      "reward": 1.8489583730697632,
+      "reward_std": 0.0416666641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8489583134651184,
+      "step": 1251
+    },
+    {
+      "completion_length": 131.0,
+      "epoch": 1.6693333333333333,
+      "grad_norm": 1.9867651776032103,
+      "kl": 0.054443359375,
+      "learning_rate": 1.6533333333333333e-07,
+      "loss": 0.0022,
+      "reward": 1.7908854484558105,
+      "reward_std": 0.05677083134651184,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8065104484558105,
+      "step": 1252
+    },
+    {
+      "completion_length": 117.328125,
+      "epoch": 1.6706666666666665,
+      "grad_norm": 1.7909623473978857,
+      "kl": 0.07373046875,
+      "learning_rate": 1.6466666666666666e-07,
+      "loss": 0.0029,
+      "reward": 1.838281273841858,
+      "reward_std": 0.026208598166704178,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8382812738418579,
+      "step": 1253
+    },
+    {
+      "completion_length": 116.53125,
+      "epoch": 1.6720000000000002,
+      "grad_norm": 2.126077171764541,
+      "kl": 0.02978515625,
+      "learning_rate": 1.64e-07,
+      "loss": 0.0012,
+      "reward": 1.8703869581222534,
+      "reward_std": 0.11130470037460327,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8703869581222534,
+      "step": 1254
+    },
+    {
+      "completion_length": 111.984375,
+      "epoch": 1.6733333333333333,
+      "grad_norm": 1.5442494346488662,
+      "kl": 0.052734375,
+      "learning_rate": 1.6333333333333331e-07,
+      "loss": 0.0021,
+      "reward": 1.8820312023162842,
+      "reward_std": 0.06530849635601044,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.882031261920929,
+      "step": 1255
+    },
+    {
+      "completion_length": 120.453125,
+      "epoch": 1.6746666666666665,
+      "grad_norm": 0.9582565983527138,
+      "kl": 0.036865234375,
+      "learning_rate": 1.6266666666666664e-07,
+      "loss": 0.0015,
+      "reward": 1.796875,
+      "reward_std": 0.12983438372612,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8125,
+      "step": 1256
+    },
+    {
+      "completion_length": 121.296875,
+      "epoch": 1.6760000000000002,
+      "grad_norm": 1.688529829820073,
+      "kl": 0.046875,
+      "learning_rate": 1.62e-07,
+      "loss": 0.0019,
+      "reward": 1.76953125,
+      "reward_std": 0.07891666889190674,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.76953125,
+      "step": 1257
+    },
+    {
+      "completion_length": 117.0625,
+      "epoch": 1.6773333333333333,
+      "grad_norm": 1.3298424790264496,
+      "kl": 0.058837890625,
+      "learning_rate": 1.6133333333333332e-07,
+      "loss": 0.0024,
+      "reward": 1.9270832538604736,
+      "reward_std": 0.0189543254673481,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9270833730697632,
+      "step": 1258
+    },
+    {
+      "completion_length": 113.0625,
+      "epoch": 1.6786666666666665,
+      "grad_norm": 1.3953171931459438,
+      "kl": 0.05908203125,
+      "learning_rate": 1.6066666666666668e-07,
+      "loss": 0.0024,
+      "reward": 1.896093726158142,
+      "reward_std": 0.02125604636967182,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8960938453674316,
+      "step": 1259
+    },
+    {
+      "completion_length": 110.296875,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 3.4681120463098107,
+      "kl": 0.07080078125,
+      "learning_rate": 1.6e-07,
+      "loss": 0.0028,
+      "reward": 1.7330728769302368,
+      "reward_std": 0.12357939779758453,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7486978769302368,
+      "step": 1260
+    },
+    {
+      "completion_length": 115.15625,
+      "epoch": 1.6813333333333333,
+      "grad_norm": 0.6456050583701076,
+      "kl": 0.0311279296875,
+      "learning_rate": 1.5933333333333333e-07,
+      "loss": 0.0012,
+      "reward": 1.8697917461395264,
+      "reward_std": 0.010416664183139801,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8697916865348816,
+      "step": 1261
+    },
+    {
+      "completion_length": 119.234375,
+      "epoch": 1.6826666666666665,
+      "grad_norm": 1.6737551068018928,
+      "kl": 0.052001953125,
+      "learning_rate": 1.5866666666666666e-07,
+      "loss": 0.0021,
+      "reward": 1.767187476158142,
+      "reward_std": 0.0320914201438427,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7671874761581421,
+      "step": 1262
+    },
+    {
+      "completion_length": 117.5625,
+      "epoch": 1.6840000000000002,
+      "grad_norm": 12.812303791164458,
+      "kl": 0.0625,
+      "learning_rate": 1.5799999999999999e-07,
+      "loss": 0.0025,
+      "reward": 1.89453125,
+      "reward_std": 0.0855635553598404,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8945313096046448,
+      "step": 1263
+    },
+    {
+      "completion_length": 113.640625,
+      "epoch": 1.6853333333333333,
+      "grad_norm": 0.7601314121550634,
+      "kl": 0.042724609375,
+      "learning_rate": 1.573333333333333e-07,
+      "loss": 0.0017,
+      "reward": 1.9739583730697632,
+      "reward_std": 0.043278127908706665,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9739583730697632,
+      "step": 1264
+    },
+    {
+      "completion_length": 125.59375,
+      "epoch": 1.6866666666666665,
+      "grad_norm": 2.975604391043326,
+      "kl": 0.061279296875,
+      "learning_rate": 1.5666666666666667e-07,
+      "loss": 0.0025,
+      "reward": 1.726822853088379,
+      "reward_std": 0.14028099179267883,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7268229126930237,
+      "step": 1265
+    },
+    {
+      "completion_length": 133.078125,
+      "epoch": 1.688,
+      "grad_norm": 0.9182757233914206,
+      "kl": 0.0458984375,
+      "learning_rate": 1.56e-07,
+      "loss": 0.0018,
+      "reward": 1.8947917222976685,
+      "reward_std": 0.07596687972545624,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9104167222976685,
+      "step": 1266
+    },
+    {
+      "completion_length": 124.75,
+      "epoch": 1.6893333333333334,
+      "grad_norm": 2.0087218562530196,
+      "kl": 0.091796875,
+      "learning_rate": 1.5533333333333332e-07,
+      "loss": 0.0037,
+      "reward": 1.7893229722976685,
+      "reward_std": 0.1276259869337082,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8205729722976685,
+      "step": 1267
+    },
+    {
+      "completion_length": 119.0,
+      "epoch": 1.6906666666666665,
+      "grad_norm": 0.665063137961419,
+      "kl": 0.03271484375,
+      "learning_rate": 1.5466666666666668e-07,
+      "loss": 0.0013,
+      "reward": 1.83984375,
+      "reward_std": 0.0078125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.83984375,
+      "step": 1268
+    },
+    {
+      "completion_length": 120.109375,
+      "epoch": 1.692,
+      "grad_norm": 1.1810456377224514,
+      "kl": 0.044677734375,
+      "learning_rate": 1.54e-07,
+      "loss": 0.0018,
+      "reward": 1.888281226158142,
+      "reward_std": 0.12968750298023224,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9039062261581421,
+      "step": 1269
+    },
+    {
+      "completion_length": 118.609375,
+      "epoch": 1.6933333333333334,
+      "grad_norm": 2.238936519484721,
+      "kl": 0.029296875,
+      "learning_rate": 1.533333333333333e-07,
+      "loss": 0.0012,
+      "reward": 1.804947853088379,
+      "reward_std": 0.06718750298023224,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8049479722976685,
+      "step": 1270
+    },
+    {
+      "completion_length": 109.984375,
+      "epoch": 1.6946666666666665,
+      "grad_norm": 1.8723715941079708,
+      "kl": 0.049072265625,
+      "learning_rate": 1.5266666666666666e-07,
+      "loss": 0.002,
+      "reward": 1.6744792461395264,
+      "reward_std": 0.08979278802871704,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.6744792461395264,
+      "step": 1271
+    },
+    {
+      "completion_length": 126.859375,
+      "epoch": 1.696,
+      "grad_norm": 1.6150874988324195,
+      "kl": 0.0380859375,
+      "learning_rate": 1.5199999999999998e-07,
+      "loss": 0.0015,
+      "reward": 1.8729166984558105,
+      "reward_std": 0.13466878235340118,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8885416984558105,
+      "step": 1272
+    },
+    {
+      "completion_length": 124.46875,
+      "epoch": 1.6973333333333334,
+      "grad_norm": 1.1388340952352503,
+      "kl": 0.032958984375,
+      "learning_rate": 1.513333333333333e-07,
+      "loss": 0.0013,
+      "reward": 1.8541667461395264,
+      "reward_std": 0.1041666641831398,
+      "rewards/format_reward": 0.953125,
+      "rewards/iou_reward": 0.9010416865348816,
+      "step": 1273
+    },
+    {
+      "completion_length": 124.453125,
+      "epoch": 1.6986666666666665,
+      "grad_norm": 1.623402265019514,
+      "kl": 0.041748046875,
+      "learning_rate": 1.5066666666666667e-07,
+      "loss": 0.0017,
+      "reward": 1.8872395753860474,
+      "reward_std": 0.09078346192836761,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8872395753860474,
+      "step": 1274
+    },
+    {
+      "completion_length": 127.203125,
+      "epoch": 1.7,
+      "grad_norm": 1.4032282139625867,
+      "kl": 0.0279541015625,
+      "learning_rate": 1.5e-07,
+      "loss": 0.0011,
+      "reward": 1.8645833730697632,
+      "reward_std": 0.053694792091846466,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8645833730697632,
+      "step": 1275
+    },
+    {
+      "completion_length": 116.984375,
+      "epoch": 1.7013333333333334,
+      "grad_norm": 2.8714101729427544,
+      "kl": 0.051513671875,
+      "learning_rate": 1.4933333333333335e-07,
+      "loss": 0.0021,
+      "reward": 1.8815104961395264,
+      "reward_std": 0.06814749538898468,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8815104365348816,
+      "step": 1276
+    },
+    {
+      "completion_length": 110.34375,
+      "epoch": 1.7026666666666666,
+      "grad_norm": 1.4982610210254566,
+      "kl": 0.04833984375,
+      "learning_rate": 1.4866666666666667e-07,
+      "loss": 0.0019,
+      "reward": 1.828125,
+      "reward_std": 0.09375,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.828125,
+      "step": 1277
+    },
+    {
+      "completion_length": 118.78125,
+      "epoch": 1.704,
+      "grad_norm": 1.2328626655429424,
+      "kl": 0.03466796875,
+      "learning_rate": 1.4799999999999998e-07,
+      "loss": 0.0014,
+      "reward": 1.8757812976837158,
+      "reward_std": 0.0728251188993454,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8757812976837158,
+      "step": 1278
+    },
+    {
+      "completion_length": 120.671875,
+      "epoch": 1.7053333333333334,
+      "grad_norm": 1.2938518866000994,
+      "kl": 0.057861328125,
+      "learning_rate": 1.4733333333333333e-07,
+      "loss": 0.0023,
+      "reward": 1.8408482074737549,
+      "reward_std": 0.11528895050287247,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8564732074737549,
+      "step": 1279
+    },
+    {
+      "completion_length": 122.890625,
+      "epoch": 1.7066666666666666,
+      "grad_norm": 1.0729190223387384,
+      "kl": 0.03515625,
+      "learning_rate": 1.4666666666666666e-07,
+      "loss": 0.0014,
+      "reward": 1.8351562023162842,
+      "reward_std": 0.0078125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.835156261920929,
+      "step": 1280
+    },
+    {
+      "completion_length": 119.78125,
+      "epoch": 1.708,
+      "grad_norm": 0.9873278897674082,
+      "kl": 0.0274658203125,
+      "learning_rate": 1.4599999999999998e-07,
+      "loss": 0.0011,
+      "reward": 1.8385417461395264,
+      "reward_std": 0.10459845513105392,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8541666865348816,
+      "step": 1281
+    },
+    {
+      "completion_length": 113.203125,
+      "epoch": 1.7093333333333334,
+      "grad_norm": 1.1979341029903405,
+      "kl": 0.05078125,
+      "learning_rate": 1.4533333333333334e-07,
+      "loss": 0.002,
+      "reward": 1.8041666746139526,
+      "reward_std": 0.12083333730697632,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8197916746139526,
+      "step": 1282
+    },
+    {
+      "completion_length": 111.453125,
+      "epoch": 1.7106666666666666,
+      "grad_norm": 2.31045400274379,
+      "kl": 0.07470703125,
+      "learning_rate": 1.4466666666666667e-07,
+      "loss": 0.003,
+      "reward": 1.9453125,
+      "reward_std": 0.0260416641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.9453125596046448,
+      "step": 1283
+    },
+    {
+      "completion_length": 133.234375,
+      "epoch": 1.712,
+      "grad_norm": 1.2792292869597894,
+      "kl": 0.04345703125,
+      "learning_rate": 1.44e-07,
+      "loss": 0.0017,
+      "reward": 1.8330729007720947,
+      "reward_std": 0.17144441604614258,
+      "rewards/format_reward": 0.96875,
+      "rewards/iou_reward": 0.8643229603767395,
+      "step": 1284
+    },
+    {
+      "completion_length": 122.859375,
+      "epoch": 1.7133333333333334,
+      "grad_norm": 1.6550285221233252,
+      "kl": 0.03466796875,
+      "learning_rate": 1.4333333333333335e-07,
+      "loss": 0.0014,
+      "reward": 1.9119791984558105,
+      "reward_std": 0.08478111028671265,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9276041388511658,
+      "step": 1285
+    },
+    {
+      "completion_length": 125.609375,
+      "epoch": 1.7146666666666666,
+      "grad_norm": 1.203478913093902,
+      "kl": 0.04736328125,
+      "learning_rate": 1.4266666666666665e-07,
+      "loss": 0.0019,
+      "reward": 1.7291667461395264,
+      "reward_std": 0.09858439117670059,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7291666269302368,
+      "step": 1286
+    },
+    {
+      "completion_length": 116.734375,
+      "epoch": 1.716,
+      "grad_norm": 1.1932304281781683,
+      "kl": 0.042724609375,
+      "learning_rate": 1.4199999999999997e-07,
+      "loss": 0.0017,
+      "reward": 1.773958444595337,
+      "reward_std": 0.05513354390859604,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7895833253860474,
+      "step": 1287
+    },
+    {
+      "completion_length": 122.96875,
+      "epoch": 1.7173333333333334,
+      "grad_norm": 1.7513151541462444,
+      "kl": 0.049072265625,
+      "learning_rate": 1.4133333333333333e-07,
+      "loss": 0.002,
+      "reward": 1.771875023841858,
+      "reward_std": 0.08760087192058563,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7875000238418579,
+      "step": 1288
+    },
+    {
+      "completion_length": 110.3125,
+      "epoch": 1.7186666666666666,
+      "grad_norm": 1.2166213573073075,
+      "kl": 0.057861328125,
+      "learning_rate": 1.4066666666666666e-07,
+      "loss": 0.0023,
+      "reward": 1.917708396911621,
+      "reward_std": 0.07150105386972427,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.9333333969116211,
+      "step": 1289
+    },
+    {
+      "completion_length": 124.328125,
+      "epoch": 1.72,
+      "grad_norm": 1.5807901251703063,
+      "kl": 0.052001953125,
+      "learning_rate": 1.4e-07,
+      "loss": 0.0021,
+      "reward": 1.8491443395614624,
+      "reward_std": 0.10360321402549744,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8491443395614624,
+      "step": 1290
+    },
+    {
+      "completion_length": 113.5625,
+      "epoch": 1.7213333333333334,
+      "grad_norm": 1.952079824923896,
+      "kl": 0.0277099609375,
+      "learning_rate": 1.3933333333333334e-07,
+      "loss": 0.0011,
+      "reward": 1.7578125,
+      "reward_std": 0.05332084745168686,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7734375,
+      "step": 1291
+    },
+    {
+      "completion_length": 111.453125,
+      "epoch": 1.7226666666666666,
+      "grad_norm": 2.647640114294708,
+      "kl": 0.0400390625,
+      "learning_rate": 1.3866666666666666e-07,
+      "loss": 0.0016,
+      "reward": 1.8958333730697632,
+      "reward_std": 0.04650106281042099,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8958333730697632,
+      "step": 1292
+    },
+    {
+      "completion_length": 127.359375,
+      "epoch": 1.724,
+      "grad_norm": 1.3245554169664517,
+      "kl": 0.0576171875,
+      "learning_rate": 1.3800000000000002e-07,
+      "loss": 0.0023,
+      "reward": 1.8341145515441895,
+      "reward_std": 0.09716348350048065,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.8497396111488342,
+      "step": 1293
+    },
+    {
+      "completion_length": 121.140625,
+      "epoch": 1.7253333333333334,
+      "grad_norm": 1.1984937411248495,
+      "kl": 0.047607421875,
+      "learning_rate": 1.3733333333333332e-07,
+      "loss": 0.0019,
+      "reward": 1.7786458730697632,
+      "reward_std": 0.02527708187699318,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.7786458730697632,
+      "step": 1294
+    },
+    {
+      "completion_length": 118.546875,
+      "epoch": 1.7266666666666666,
+      "grad_norm": 2.3589363834986496,
+      "kl": 0.043701171875,
+      "learning_rate": 1.3666666666666665e-07,
+      "loss": 0.0017,
+      "reward": 1.8640625476837158,
+      "reward_std": 0.16904377937316895,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8640625476837158,
+      "step": 1295
+    },
+    {
+      "completion_length": 124.5,
+      "epoch": 1.728,
+      "grad_norm": 1.5080434726320788,
+      "kl": 0.048583984375,
+      "learning_rate": 1.36e-07,
+      "loss": 0.0019,
+      "reward": 1.7739211320877075,
+      "reward_std": 0.04025297239422798,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.7895461320877075,
+      "step": 1296
+    },
+    {
+      "completion_length": 108.1875,
+      "epoch": 1.7293333333333334,
+      "grad_norm": 0.6540875097828674,
+      "kl": 0.02783203125,
+      "learning_rate": 1.3533333333333333e-07,
+      "loss": 0.0011,
+      "reward": 1.8802083730697632,
+      "reward_std": 0.03125,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8802083134651184,
+      "step": 1297
+    },
+    {
+      "completion_length": 113.390625,
+      "epoch": 1.7306666666666666,
+      "grad_norm": 1.919968567186987,
+      "kl": 0.0380859375,
+      "learning_rate": 1.3466666666666665e-07,
+      "loss": 0.0015,
+      "reward": 1.8802083730697632,
+      "reward_std": 0.09182828664779663,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8802083730697632,
+      "step": 1298
+    },
+    {
+      "completion_length": 119.484375,
+      "epoch": 1.732,
+      "grad_norm": 1.1309148575746413,
+      "kl": 0.05419921875,
+      "learning_rate": 1.34e-07,
+      "loss": 0.0022,
+      "reward": 1.87109375,
+      "reward_std": 0.0494791641831398,
+      "rewards/format_reward": 1.0,
+      "rewards/iou_reward": 0.8710938096046448,
+      "step": 1299
+    },
+    {
+      "completion_length": 120.796875,
+      "epoch": 1.7333333333333334,
+      "grad_norm": 1.2427872853930964,
+      "kl": 0.04248046875,
+      "learning_rate": 1.3333333333333334e-07,
+      "loss": 0.0017,
+      "reward": 1.9031250476837158,
+      "reward_std": 0.04193740338087082,
+      "rewards/format_reward": 0.984375,
+      "rewards/iou_reward": 0.918749988079071,
+      "step": 1300
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 1500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}