diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,4233 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5141388174807198,
+  "eval_steps": 500,
+  "global_step": 300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2889.4444580078125,
+      "epoch": 0.001713796058269066,
+      "grad_norm": 0.1318138986825943,
+      "kl": 0.0,
+      "learning_rate": 1.6949152542372882e-08,
+      "loss": 0.0468,
+      "reward": 0.861111119389534,
+      "reward_std": 0.38495802134275436,
+      "rewards/accuracy_reward": 0.20833333674818277,
+      "rewards/format_reward": 0.4444444440305233,
+      "step": 1
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2912.0833129882812,
+      "epoch": 0.003427592116538132,
+      "grad_norm": 0.07943949103355408,
+      "kl": 0.0,
+      "learning_rate": 3.3898305084745764e-08,
+      "loss": 0.0335,
+      "reward": 0.47222224064171314,
+      "reward_std": 0.1791159175336361,
+      "rewards/accuracy_reward": 0.0694444477558136,
+      "rewards/format_reward": 0.33333334513008595,
+      "step": 2
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2928.4444580078125,
+      "epoch": 0.005141388174807198,
+      "grad_norm": 0.09856246411800385,
+      "kl": 3.4332275390625e-05,
+      "learning_rate": 5.0847457627118645e-08,
+      "loss": 0.0448,
+      "reward": 0.6388888880610466,
+      "reward_std": 0.23899273201823235,
+      "rewards/accuracy_reward": 0.13888889271765947,
+      "rewards/format_reward": 0.3611111156642437,
+      "step": 3
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2919.0555419921875,
+      "epoch": 0.006855184233076264,
+      "grad_norm": 0.13336561620235443,
+      "kl": 3.319978713989258e-05,
+      "learning_rate": 6.779661016949153e-08,
+      "loss": 0.0937,
+      "reward": 1.0416666865348816,
+      "reward_std": 0.8419074863195419,
+      "rewards/accuracy_reward": 0.3055555634200573,
+      "rewards/format_reward": 0.4305555671453476,
+      "step": 4
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2756.0000610351562,
+      "epoch": 0.00856898029134533,
+      "grad_norm": 0.20592202246189117,
+      "kl": 1.940131187438965e-05,
+      "learning_rate": 8.47457627118644e-08,
+      "loss": 0.1041,
+      "reward": 0.986111119389534,
+      "reward_std": 0.4405064880847931,
+      "rewards/accuracy_reward": 0.23611112032085657,
+      "rewards/format_reward": 0.5138888955116272,
+      "step": 5
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3220.4166870117188,
+      "epoch": 0.010282776349614395,
+      "grad_norm": 0.15423689782619476,
+      "kl": 3.999471664428711e-05,
+      "learning_rate": 1.0169491525423729e-07,
+      "loss": 0.0714,
+      "reward": 0.5972222350537777,
+      "reward_std": 0.6300827041268349,
+      "rewards/accuracy_reward": 0.13888888992369175,
+      "rewards/format_reward": 0.31944444589316845,
+      "step": 6
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2272.7222595214844,
+      "epoch": 0.011996572407883462,
+      "grad_norm": 0.1403103917837143,
+      "kl": 3.629922866821289e-05,
+      "learning_rate": 1.1864406779661017e-07,
+      "loss": 0.0274,
+      "reward": 1.0972222238779068,
+      "reward_std": 0.3974733129143715,
+      "rewards/accuracy_reward": 0.2500000074505806,
+      "rewards/format_reward": 0.5972222238779068,
+      "step": 7
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3010.4722290039062,
+      "epoch": 0.013710368466152529,
+      "grad_norm": 0.15772269666194916,
+      "kl": 2.4080276489257812e-05,
+      "learning_rate": 1.3559322033898305e-07,
+      "loss": 0.0218,
+      "reward": 1.3888889253139496,
+      "reward_std": 0.8940589390695095,
+      "rewards/accuracy_reward": 0.430555559694767,
+      "rewards/format_reward": 0.5277777835726738,
+      "step": 8
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2840.0556030273438,
+      "epoch": 0.015424164524421594,
+      "grad_norm": 0.10687297582626343,
+      "kl": 2.6017427444458008e-05,
+      "learning_rate": 1.5254237288135593e-07,
+      "loss": 0.0778,
+      "reward": 0.9305555522441864,
+      "reward_std": 0.6126390919089317,
+      "rewards/accuracy_reward": 0.2500000037252903,
+      "rewards/format_reward": 0.43055555783212185,
+      "step": 9
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2991.7083129882812,
+      "epoch": 0.01713796058269066,
+      "grad_norm": 0.15705247223377228,
+      "kl": 3.451108932495117e-05,
+      "learning_rate": 1.694915254237288e-07,
+      "loss": 0.0238,
+      "reward": 0.5833333283662796,
+      "reward_std": 0.6530618071556091,
+      "rewards/accuracy_reward": 0.15277777891606092,
+      "rewards/format_reward": 0.27777778171002865,
+      "step": 10
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2877.375030517578,
+      "epoch": 0.018851756640959727,
+      "grad_norm": 0.21513371169567108,
+      "kl": 3.460049629211426e-05,
+      "learning_rate": 1.8644067796610168e-07,
+      "loss": 0.0466,
+      "reward": 0.7916666800156236,
+      "reward_std": 0.18172631040215492,
+      "rewards/accuracy_reward": 0.2361111119389534,
+      "rewards/format_reward": 0.3194444412365556,
+      "step": 11
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3343.013916015625,
+      "epoch": 0.02056555269922879,
+      "grad_norm": 0.14181174337863922,
+      "kl": 3.936886787414551e-05,
+      "learning_rate": 2.0338983050847458e-07,
+      "loss": 0.074,
+      "reward": 0.3888888908550143,
+      "reward_std": 0.5541914477944374,
+      "rewards/accuracy_reward": 0.08333333395421505,
+      "rewards/format_reward": 0.22222222667187452,
+      "step": 12
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2895.25,
+      "epoch": 0.022279348757497857,
+      "grad_norm": 0.12315575033426285,
+      "kl": 3.3020973205566406e-05,
+      "learning_rate": 2.2033898305084743e-07,
+      "loss": 0.0576,
+      "reward": 0.833333358168602,
+      "reward_std": 0.5088144801557064,
+      "rewards/accuracy_reward": 0.18055555783212185,
+      "rewards/format_reward": 0.4722222201526165,
+      "step": 13
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3024.763916015625,
+      "epoch": 0.023993144815766924,
+      "grad_norm": 0.1697244644165039,
+      "kl": 2.4959444999694824e-05,
+      "learning_rate": 2.3728813559322033e-07,
+      "loss": 0.0401,
+      "reward": 0.666666679084301,
+      "reward_std": 0.29541126638650894,
+      "rewards/accuracy_reward": 0.18055556062608957,
+      "rewards/format_reward": 0.3055555634200573,
+      "step": 14
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3397.7916259765625,
+      "epoch": 0.02570694087403599,
+      "grad_norm": 0.14056538045406342,
+      "kl": 4.1157007217407227e-05,
+      "learning_rate": 2.542372881355932e-07,
+      "loss": 0.0871,
+      "reward": 0.33333333767950535,
+      "reward_std": 0.616430725902319,
+      "rewards/accuracy_reward": 0.08333333488553762,
+      "rewards/format_reward": 0.16666666977107525,
+      "step": 15
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2442.4722595214844,
+      "epoch": 0.027420736932305057,
+      "grad_norm": 0.15735530853271484,
+      "kl": 2.2862106561660767e-05,
+      "learning_rate": 2.711864406779661e-07,
+      "loss": 0.0156,
+      "reward": 1.4166666567325592,
+      "reward_std": 0.86329685151577,
+      "rewards/accuracy_reward": 0.4305555634200573,
+      "rewards/format_reward": 0.5555555522441864,
+      "step": 16
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2926.25,
+      "epoch": 0.02913453299057412,
+      "grad_norm": 0.0811009407043457,
+      "kl": 2.7507543563842773e-05,
+      "learning_rate": 2.88135593220339e-07,
+      "loss": 0.0173,
+      "reward": 0.8750000167638063,
+      "reward_std": 0.2480051852762699,
+      "rewards/accuracy_reward": 0.2777777798473835,
+      "rewards/format_reward": 0.31944445334374905,
+      "step": 17
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3358.9722900390625,
+      "epoch": 0.030848329048843187,
+      "grad_norm": 0.1565597802400589,
+      "kl": 3.2454729080200195e-05,
+      "learning_rate": 3.0508474576271186e-07,
+      "loss": 0.0761,
+      "reward": 0.3472222350537777,
+      "reward_std": 0.47978880628943443,
+      "rewards/accuracy_reward": 0.0972222238779068,
+      "rewards/format_reward": 0.1527777798473835,
+      "step": 18
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3248.263916015625,
+      "epoch": 0.032562125107112254,
+      "grad_norm": 0.06823915243148804,
+      "kl": 3.1620264053344727e-05,
+      "learning_rate": 3.220338983050847e-07,
+      "loss": 0.0115,
+      "reward": 0.611111119389534,
+      "reward_std": 0.4586857333779335,
+      "rewards/accuracy_reward": 0.18055555690079927,
+      "rewards/format_reward": 0.2500000037252903,
+      "step": 19
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3367.013916015625,
+      "epoch": 0.03427592116538132,
+      "grad_norm": 0.09324803203344345,
+      "kl": 3.3974647521972656e-05,
+      "learning_rate": 3.389830508474576e-07,
+      "loss": 0.0378,
+      "reward": 0.5,
+      "reward_std": 0.47972556948661804,
+      "rewards/accuracy_reward": 0.1527777798473835,
+      "rewards/format_reward": 0.1944444514811039,
+      "step": 20
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3031.4583740234375,
+      "epoch": 0.03598971722365039,
+      "grad_norm": 0.11860450357198715,
+      "kl": 4.3004751205444336e-05,
+      "learning_rate": 3.559322033898305e-07,
+      "loss": 0.0817,
+      "reward": 0.541666679084301,
+      "reward_std": 0.5936876386404037,
+      "rewards/accuracy_reward": 0.12500000186264515,
+      "rewards/format_reward": 0.2916666716337204,
+      "step": 21
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3335.4444580078125,
+      "epoch": 0.037703513281919454,
+      "grad_norm": 0.09448128193616867,
+      "kl": 2.7835369110107422e-05,
+      "learning_rate": 3.7288135593220336e-07,
+      "loss": 0.0319,
+      "reward": 0.5138888908550143,
+      "reward_std": 0.37944628670811653,
+      "rewards/accuracy_reward": 0.1388888917863369,
+      "rewards/format_reward": 0.23611111473292112,
+      "step": 22
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3147.4583129882812,
+      "epoch": 0.03941730934018852,
+      "grad_norm": 0.11545193940401077,
+      "kl": 2.8133392333984375e-05,
+      "learning_rate": 3.898305084745763e-07,
+      "loss": 0.0274,
+      "reward": 0.4166666753590107,
+      "reward_std": 0.4431168958544731,
+      "rewards/accuracy_reward": 0.08333333488553762,
+      "rewards/format_reward": 0.2500000046566129,
+      "step": 23
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2804.5416870117188,
+      "epoch": 0.04113110539845758,
+      "grad_norm": 0.22444979846477509,
+      "kl": 2.9146671295166016e-05,
+      "learning_rate": 4.0677966101694916e-07,
+      "loss": 0.0787,
+      "reward": 1.0416666716337204,
+      "reward_std": 0.23116151615977287,
+      "rewards/accuracy_reward": 0.3055555634200573,
+      "rewards/format_reward": 0.4305555671453476,
+      "step": 24
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3022.1527709960938,
+      "epoch": 0.04284490145672665,
+      "grad_norm": 0.11472934484481812,
+      "kl": 3.597140312194824e-05,
+      "learning_rate": 4.23728813559322e-07,
+      "loss": 0.0722,
+      "reward": 0.7638888955116272,
+      "reward_std": 0.4787874035537243,
+      "rewards/accuracy_reward": 0.1944444477558136,
+      "rewards/format_reward": 0.3750000149011612,
+      "step": 25
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3167.3195190429688,
+      "epoch": 0.044558697514995714,
+      "grad_norm": 0.23513932526111603,
+      "kl": 3.221631050109863e-05,
+      "learning_rate": 4.4067796610169486e-07,
+      "loss": 0.1099,
+      "reward": 0.5000000186264515,
+      "reward_std": 0.6264622695744038,
+      "rewards/accuracy_reward": 0.11111111473292112,
+      "rewards/format_reward": 0.27777778916060925,
+      "step": 26
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3249.5833129882812,
+      "epoch": 0.04627249357326478,
+      "grad_norm": 0.10312272608280182,
+      "kl": 1.971423625946045e-05,
+      "learning_rate": 4.576271186440678e-07,
+      "loss": 0.036,
+      "reward": 0.6250000037252903,
+      "reward_std": 0.3617309741675854,
+      "rewards/accuracy_reward": 0.15277778077870607,
+      "rewards/format_reward": 0.3194444514811039,
+      "step": 27
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2603.77783203125,
+      "epoch": 0.04798628963153385,
+      "grad_norm": 0.15025873482227325,
+      "kl": 2.3640692234039307e-05,
+      "learning_rate": 4.7457627118644066e-07,
+      "loss": 0.0584,
+      "reward": 1.1111111342906952,
+      "reward_std": 0.4762897342443466,
+      "rewards/accuracy_reward": 0.3055555634200573,
+      "rewards/format_reward": 0.5000000149011612,
+      "step": 28
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3161.8472900390625,
+      "epoch": 0.049700085689802914,
+      "grad_norm": 0.12228947132825851,
+      "kl": 3.4302473068237305e-05,
+      "learning_rate": 4.915254237288136e-07,
+      "loss": 0.0351,
+      "reward": 0.8333333302289248,
+      "reward_std": 0.6312860958278179,
+      "rewards/accuracy_reward": 0.236111119389534,
+      "rewards/format_reward": 0.36111110635101795,
+      "step": 29
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2754.902801513672,
+      "epoch": 0.05141388174807198,
+      "grad_norm": 0.24735864996910095,
+      "kl": 1.800060272216797e-05,
+      "learning_rate": 5.084745762711864e-07,
+      "loss": 0.1167,
+      "reward": 1.2638889253139496,
+      "reward_std": 0.8531928509473801,
+      "rewards/accuracy_reward": 0.361111119389534,
+      "rewards/format_reward": 0.541666679084301,
+      "step": 30
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3002.125,
+      "epoch": 0.05312767780634105,
+      "grad_norm": 0.15179027616977692,
+      "kl": 9.566545486450195e-06,
+      "learning_rate": 5.254237288135593e-07,
+      "loss": 0.0781,
+      "reward": 0.6388889010995626,
+      "reward_std": 0.28320711851119995,
+      "rewards/accuracy_reward": 0.180555559694767,
+      "rewards/format_reward": 0.27777778171002865,
+      "step": 31
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3020.0694580078125,
+      "epoch": 0.054841473864610114,
+      "grad_norm": 0.2012336701154709,
+      "kl": 2.09808349609375e-05,
+      "learning_rate": 5.423728813559322e-07,
+      "loss": 0.1265,
+      "reward": 0.9861111342906952,
+      "reward_std": 0.9004222899675369,
+      "rewards/accuracy_reward": 0.26388888992369175,
+      "rewards/format_reward": 0.4583333395421505,
+      "step": 32
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2485.0555725097656,
+      "epoch": 0.056555269922879174,
+      "grad_norm": 0.13532550632953644,
+      "kl": 9.387731552124023e-06,
+      "learning_rate": 5.59322033898305e-07,
+      "loss": 0.0014,
+      "reward": 1.1250000149011612,
+      "reward_std": 0.6304849497973919,
+      "rewards/accuracy_reward": 0.305555559694767,
+      "rewards/format_reward": 0.5138889029622078,
+      "step": 33
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2882.5833740234375,
+      "epoch": 0.05826906598114824,
+      "grad_norm": 0.11332134157419205,
+      "kl": 1.0229647159576416e-05,
+      "learning_rate": 5.76271186440678e-07,
+      "loss": 0.0378,
+      "reward": 0.847222238779068,
+      "reward_std": 0.5727614238858223,
+      "rewards/accuracy_reward": 0.2500000074505806,
+      "rewards/format_reward": 0.3472222276031971,
+      "step": 34
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3271.638916015625,
+      "epoch": 0.05998286203941731,
+      "grad_norm": 0.10583982616662979,
+      "kl": 2.9146671295166016e-05,
+      "learning_rate": 5.932203389830508e-07,
+      "loss": 0.0459,
+      "reward": 0.37500000558793545,
+      "reward_std": 0.22475946694612503,
+      "rewards/accuracy_reward": 0.0694444477558136,
+      "rewards/format_reward": 0.23611111752688885,
+      "step": 35
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2812.0833740234375,
+      "epoch": 0.061696658097686374,
+      "grad_norm": 0.17242440581321716,
+      "kl": 5.033612251281738e-05,
+      "learning_rate": 6.101694915254237e-07,
+      "loss": 0.086,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.25616974383592606,
+      "rewards/accuracy_reward": 0.0694444477558136,
+      "rewards/format_reward": 0.3194444477558136,
+      "step": 36
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3051.763916015625,
+      "epoch": 0.06341045415595545,
+      "grad_norm": 0.13607284426689148,
+      "kl": 9.356439113616943e-05,
+      "learning_rate": 6.271186440677966e-07,
+      "loss": 0.0796,
+      "reward": 0.4861111231148243,
+      "reward_std": 0.4781397730112076,
+      "rewards/accuracy_reward": 0.12500000279396772,
+      "rewards/format_reward": 0.23611111752688885,
+      "step": 37
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3268.375,
+      "epoch": 0.06512425021422451,
+      "grad_norm": 0.10063257068395615,
+      "kl": 2.215057611465454e-05,
+      "learning_rate": 6.440677966101694e-07,
+      "loss": 0.075,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.4407730996608734,
+      "rewards/accuracy_reward": 0.0972222238779068,
+      "rewards/format_reward": 0.22222222946584225,
+      "step": 38
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3201.2083129882812,
+      "epoch": 0.06683804627249357,
+      "grad_norm": 0.0791403129696846,
+      "kl": 4.489719867706299e-05,
+      "learning_rate": 6.610169491525423e-07,
+      "loss": 0.028,
+      "reward": 0.5138889029622078,
+      "reward_std": 0.3695859834551811,
+      "rewards/accuracy_reward": 0.13888889085501432,
+      "rewards/format_reward": 0.23611111380159855,
+      "step": 39
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2954.5138549804688,
+      "epoch": 0.06855184233076264,
+      "grad_norm": 0.1909833401441574,
+      "kl": 8.817017078399658e-05,
+      "learning_rate": 6.779661016949152e-07,
+      "loss": 0.0638,
+      "reward": 0.8750000149011612,
+      "reward_std": 0.3410548157989979,
+      "rewards/accuracy_reward": 0.2777777872979641,
+      "rewards/format_reward": 0.3194444514811039,
+      "step": 40
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2738.6250610351562,
+      "epoch": 0.0702656383890317,
+      "grad_norm": 0.12753309309482574,
+      "kl": 3.444775938987732e-05,
+      "learning_rate": 6.949152542372881e-07,
+      "loss": 0.0836,
+      "reward": 1.1388889104127884,
+      "reward_std": 0.6430224105715752,
+      "rewards/accuracy_reward": 0.3472222276031971,
+      "rewards/format_reward": 0.4444444477558136,
+      "step": 41
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2710.8750610351562,
+      "epoch": 0.07197943444730077,
+      "grad_norm": 0.1831730157136917,
+      "kl": 0.0002748072147369385,
+      "learning_rate": 7.11864406779661e-07,
+      "loss": 0.0698,
+      "reward": 0.9305555857717991,
+      "reward_std": 0.4780263118445873,
+      "rewards/accuracy_reward": 0.236111119389534,
+      "rewards/format_reward": 0.4583333469927311,
+      "step": 42
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2921.9861450195312,
+      "epoch": 0.07369323050556983,
+      "grad_norm": 0.15687353909015656,
+      "kl": 8.583441376686096e-05,
+      "learning_rate": 7.288135593220338e-07,
+      "loss": 0.0681,
+      "reward": 1.0277777910232544,
+      "reward_std": 0.5702280625700951,
+      "rewards/accuracy_reward": 0.2916666716337204,
+      "rewards/format_reward": 0.4444444477558136,
+      "step": 43
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3053.9166870117188,
+      "epoch": 0.07540702656383891,
+      "grad_norm": 0.16944237053394318,
+      "kl": 3.604590892791748e-05,
+      "learning_rate": 7.457627118644067e-07,
+      "loss": 0.1266,
+      "reward": 0.6527777910232544,
+      "reward_std": 0.6824748627841473,
+      "rewards/accuracy_reward": 0.13888889271765947,
+      "rewards/format_reward": 0.3750000037252903,
+      "step": 44
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3097.3194580078125,
+      "epoch": 0.07712082262210797,
+      "grad_norm": 0.12692783772945404,
+      "kl": 4.616379737854004e-05,
+      "learning_rate": 7.627118644067796e-07,
+      "loss": 0.0334,
+      "reward": 0.6944444626569748,
+      "reward_std": 0.5105769783258438,
+      "rewards/accuracy_reward": 0.16666666977107525,
+      "rewards/format_reward": 0.361111119389534,
+      "step": 45
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2547.388916015625,
+      "epoch": 0.07883461868037704,
+      "grad_norm": 0.17742452025413513,
+      "kl": 0.00026345252990722656,
+      "learning_rate": 7.796610169491526e-07,
+      "loss": 0.0976,
+      "reward": 0.6527777761220932,
+      "reward_std": 0.34457773715257645,
+      "rewards/accuracy_reward": 0.09722222574055195,
+      "rewards/format_reward": 0.4583333432674408,
+      "step": 46
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3137.90283203125,
+      "epoch": 0.0805484147386461,
+      "grad_norm": 0.15140484273433685,
+      "kl": 0.0002467818558216095,
+      "learning_rate": 7.966101694915253e-07,
+      "loss": 0.0735,
+      "reward": 0.888888917863369,
+      "reward_std": 0.7274487726390362,
+      "rewards/accuracy_reward": 0.23611111752688885,
+      "rewards/format_reward": 0.4166666753590107,
+      "step": 47
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2775.9722290039062,
+      "epoch": 0.08226221079691516,
+      "grad_norm": 0.23574158549308777,
+      "kl": 0.00021022558212280273,
+      "learning_rate": 8.135593220338983e-07,
+      "loss": 0.1454,
+      "reward": 1.2777777910232544,
+      "reward_std": 0.6754349321126938,
+      "rewards/accuracy_reward": 0.3888888955116272,
+      "rewards/format_reward": 0.5000000074505806,
+      "step": 48
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2843.5972290039062,
+      "epoch": 0.08397600685518423,
+      "grad_norm": 0.1440126597881317,
+      "kl": 0.0003483295440673828,
+      "learning_rate": 8.305084745762712e-07,
+      "loss": 0.0911,
+      "reward": 1.1250000447034836,
+      "reward_std": 0.5149499028921127,
+      "rewards/accuracy_reward": 0.3472222238779068,
+      "rewards/format_reward": 0.430555559694767,
+      "step": 49
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2877.02783203125,
+      "epoch": 0.0856898029134533,
+      "grad_norm": 0.16242656111717224,
+      "kl": 0.0005815029144287109,
+      "learning_rate": 8.47457627118644e-07,
+      "loss": 0.1154,
+      "reward": 0.7361111342906952,
+      "reward_std": 0.497927188873291,
+      "rewards/accuracy_reward": 0.1527777835726738,
+      "rewards/format_reward": 0.4305555745959282,
+      "step": 50
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3075.5833129882812,
+      "epoch": 0.08740359897172237,
+      "grad_norm": 0.09776900708675385,
+      "kl": 0.00047463178634643555,
+      "learning_rate": 8.64406779661017e-07,
+      "loss": 0.0336,
+      "reward": 0.9305555820465088,
+      "reward_std": 0.6370660364627838,
+      "rewards/accuracy_reward": 0.3055555634200573,
+      "rewards/format_reward": 0.3194444477558136,
+      "step": 51
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3223.0000610351562,
+      "epoch": 0.08911739502999143,
+      "grad_norm": 0.09731809794902802,
+      "kl": 0.00039637088775634766,
+      "learning_rate": 8.813559322033897e-07,
+      "loss": 0.0465,
+      "reward": 0.5833333358168602,
+      "reward_std": 0.5373477786779404,
+      "rewards/accuracy_reward": 0.16666666977107525,
+      "rewards/format_reward": 0.25000000931322575,
+      "step": 52
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3072.638916015625,
+      "epoch": 0.0908311910882605,
+      "grad_norm": 0.11242014914751053,
+      "kl": 0.0001506805419921875,
+      "learning_rate": 8.983050847457627e-07,
+      "loss": 0.0368,
+      "reward": 0.5694444589316845,
+      "reward_std": 0.3695859871804714,
+      "rewards/accuracy_reward": 0.09722222480922937,
+      "rewards/format_reward": 0.37500000186264515,
+      "step": 53
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3340.65283203125,
+      "epoch": 0.09254498714652956,
+      "grad_norm": 0.09306000173091888,
+      "kl": 0.00031280517578125,
+      "learning_rate": 9.152542372881356e-07,
+      "loss": 0.0025,
+      "reward": 0.6111111342906952,
+      "reward_std": 0.5472080707550049,
+      "rewards/accuracy_reward": 0.1944444477558136,
+      "rewards/format_reward": 0.2222222238779068,
+      "step": 54
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3444.361083984375,
+      "epoch": 0.09425878320479864,
+      "grad_norm": 0.10728821158409119,
+      "kl": 0.0005254745483398438,
+      "learning_rate": 9.322033898305083e-07,
+      "loss": 0.0437,
+      "reward": 0.3888888955116272,
+      "reward_std": 0.4680837541818619,
+      "rewards/accuracy_reward": 0.08333333395421505,
+      "rewards/format_reward": 0.22222222667187452,
+      "step": 55
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3232.0416870117188,
+      "epoch": 0.0959725792630677,
+      "grad_norm": 0.12119881063699722,
+      "kl": 0.00034499168395996094,
+      "learning_rate": 9.491525423728813e-07,
+      "loss": 0.0786,
+      "reward": 0.7916666828095913,
+      "reward_std": 0.4137357361614704,
+      "rewards/accuracy_reward": 0.25,
+      "rewards/format_reward": 0.2916666679084301,
+      "step": 56
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3270.388916015625,
+      "epoch": 0.09768637532133675,
+      "grad_norm": 0.11568433046340942,
+      "kl": 0.0003733634948730469,
+      "learning_rate": 9.661016949152542e-07,
+      "loss": 0.0212,
+      "reward": 0.652777798473835,
+      "reward_std": 0.6071257442235947,
+      "rewards/accuracy_reward": 0.16666666883975267,
+      "rewards/format_reward": 0.3194444552063942,
+      "step": 57
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3276.7777709960938,
+      "epoch": 0.09940017137960583,
+      "grad_norm": 0.1174827516078949,
+      "kl": 0.002979278564453125,
+      "learning_rate": 9.830508474576272e-07,
+      "loss": 0.0607,
+      "reward": 0.3611111156642437,
+      "reward_std": 0.4297382980585098,
+      "rewards/accuracy_reward": 0.06944444496184587,
+      "rewards/format_reward": 0.2222222276031971,
+      "step": 58
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3178.65283203125,
+      "epoch": 0.10111396743787489,
+      "grad_norm": 0.13718360662460327,
+      "kl": 0.00142669677734375,
+      "learning_rate": 1e-06,
+      "loss": 0.0701,
+      "reward": 0.9861111342906952,
+      "reward_std": 0.7938041463494301,
+      "rewards/accuracy_reward": 0.31944444961845875,
+      "rewards/format_reward": 0.3472222276031971,
+      "step": 59
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3385.9166259765625,
+      "epoch": 0.10282776349614396,
+      "grad_norm": 0.1017572283744812,
+      "kl": 0.0005638599395751953,
+      "learning_rate": 9.999919124237425e-07,
+      "loss": 0.0466,
+      "reward": 0.5833333283662796,
+      "reward_std": 0.5943029820919037,
+      "rewards/accuracy_reward": 0.180555559694767,
+      "rewards/format_reward": 0.22222222480922937,
+      "step": 60
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3338.388916015625,
+      "epoch": 0.10454155955441302,
+      "grad_norm": 0.1060507744550705,
+      "kl": 0.0013508796691894531,
+      "learning_rate": 9.999676499856762e-07,
+      "loss": 0.057,
+      "reward": 0.3888888880610466,
+      "reward_std": 0.2651822194457054,
+      "rewards/accuracy_reward": 0.0694444477558136,
+      "rewards/format_reward": 0.2500000074505806,
+      "step": 61
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2797.0416870117188,
+      "epoch": 0.1062553556126821,
+      "grad_norm": 0.13327208161354065,
+      "kl": 0.004306793212890625,
+      "learning_rate": 9.999272135579094e-07,
+      "loss": 0.0093,
+      "reward": 0.9722222238779068,
+      "reward_std": 0.4413543753325939,
+      "rewards/accuracy_reward": 0.2638888955116272,
+      "rewards/format_reward": 0.4444444552063942,
+      "step": 62
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3424.4861450195312,
+      "epoch": 0.10796915167095116,
+      "grad_norm": 0.057017724961042404,
+      "kl": 0.0010051727294921875,
+      "learning_rate": 9.998706045939205e-07,
+      "loss": 0.0094,
+      "reward": 0.5416666641831398,
+      "reward_std": 0.360260970890522,
+      "rewards/accuracy_reward": 0.1527777798473835,
+      "rewards/format_reward": 0.23611111380159855,
+      "step": 63
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2833.3334045410156,
+      "epoch": 0.10968294772922023,
+      "grad_norm": 0.1504010707139969,
+      "kl": 0.0013284683227539062,
+      "learning_rate": 9.997978251285065e-07,
+      "loss": 0.0029,
+      "reward": 0.6527778003364801,
+      "reward_std": 0.4071078971028328,
+      "rewards/accuracy_reward": 0.1250000037252903,
+      "rewards/format_reward": 0.40277778543531895,
+      "step": 64
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3199.5972290039062,
+      "epoch": 0.11139674378748929,
+      "grad_norm": 0.11405781656503677,
+      "kl": 0.001636505126953125,
+      "learning_rate": 9.997088777777095e-07,
+      "loss": 0.0488,
+      "reward": 0.7222222210839391,
+      "reward_std": 0.5615956597030163,
+      "rewards/accuracy_reward": 0.23611111659556627,
+      "rewards/format_reward": 0.24999999906867743,
+      "step": 65
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2915.388916015625,
+      "epoch": 0.11311053984575835,
+      "grad_norm": 0.11168493330478668,
+      "kl": 0.000568389892578125,
+      "learning_rate": 9.99603765738723e-07,
+      "loss": -0.0035,
+      "reward": 0.7777777910232544,
+      "reward_std": 0.2721655145287514,
+      "rewards/accuracy_reward": 0.23611112032085657,
+      "rewards/format_reward": 0.3055555671453476,
+      "step": 66
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3189.40283203125,
+      "epoch": 0.11482433590402742,
+      "grad_norm": 0.12389721721410751,
+      "kl": 0.001224517822265625,
+      "learning_rate": 9.994824927897762e-07,
+      "loss": 0.0351,
+      "reward": 0.6111111044883728,
+      "reward_std": 0.5111561641097069,
+      "rewards/accuracy_reward": 0.15277778450399637,
+      "rewards/format_reward": 0.3055555671453476,
+      "step": 67
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3199.2777709960938,
+      "epoch": 0.11653813196229648,
+      "grad_norm": 0.04368606582283974,
+      "kl": 0.0010256767272949219,
+      "learning_rate": 9.993450632899989e-07,
+      "loss": 0.0049,
+      "reward": 0.4722222238779068,
+      "reward_std": 0.2300211265683174,
+      "rewards/accuracy_reward": 0.12500000186264515,
+      "rewards/format_reward": 0.22222222574055195,
+      "step": 68
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3365.9583740234375,
+      "epoch": 0.11825192802056556,
+      "grad_norm": 0.10794109106063843,
+      "kl": 0.0010347366333007812,
+      "learning_rate": 9.99191482179265e-07,
+      "loss": 0.0407,
+      "reward": 0.6388889029622078,
+      "reward_std": 0.6728046834468842,
+      "rewards/accuracy_reward": 0.1944444477558136,
+      "rewards/format_reward": 0.25000000558793545,
+      "step": 69
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2993.1527709960938,
+      "epoch": 0.11996572407883462,
+      "grad_norm": 0.08366014063358307,
+      "kl": 0.0018405914306640625,
+      "learning_rate": 9.99021754978014e-07,
+      "loss": -0.0295,
+      "reward": 0.8194444589316845,
+      "reward_std": 0.4187307730317116,
+      "rewards/accuracy_reward": 0.25000000186264515,
+      "rewards/format_reward": 0.3194444486871362,
+      "step": 70
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3297.041748046875,
+      "epoch": 0.12167952013710369,
+      "grad_norm": 0.12551282346248627,
+      "kl": 0.0020809173583984375,
+      "learning_rate": 9.988358877870534e-07,
+      "loss": 0.057,
+      "reward": 0.8750000111758709,
+      "reward_std": 0.766589842736721,
+      "rewards/accuracy_reward": 0.27777778450399637,
+      "rewards/format_reward": 0.3194444486871362,
+      "step": 71
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2669.166717529297,
+      "epoch": 0.12339331619537275,
+      "grad_norm": 0.07140244543552399,
+      "kl": 0.0021390914916992188,
+      "learning_rate": 9.986338872873393e-07,
+      "loss": 0.0464,
+      "reward": 1.2638888955116272,
+      "reward_std": 0.31082576885819435,
+      "rewards/accuracy_reward": 0.3750000027939677,
+      "rewards/format_reward": 0.5138888955116272,
+      "step": 72
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3122.9166870117188,
+      "epoch": 0.12510711225364182,
+      "grad_norm": 0.1383487433195114,
+      "kl": 0.0014562606811523438,
+      "learning_rate": 9.984157607397357e-07,
+      "loss": 0.0568,
+      "reward": 0.8472222201526165,
+      "reward_std": 0.5111991167068481,
+      "rewards/accuracy_reward": 0.23611112125217915,
+      "rewards/format_reward": 0.37500000838190317,
+      "step": 73
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2634.4306030273438,
+      "epoch": 0.1268209083119109,
+      "grad_norm": 0.16660131514072418,
+      "kl": 0.00234222412109375,
+      "learning_rate": 9.981815159847542e-07,
+      "loss": 0.0609,
+      "reward": 1.2638888955116272,
+      "reward_std": 0.599253699183464,
+      "rewards/accuracy_reward": 0.3472222276031971,
+      "rewards/format_reward": 0.5694444477558136,
+      "step": 74
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3220.75,
+      "epoch": 0.12853470437017994,
+      "grad_norm": 0.05945681035518646,
+      "kl": 0.00293731689453125,
+      "learning_rate": 9.979311614422718e-07,
+      "loss": 0.0001,
+      "reward": 0.8055555671453476,
+      "reward_std": 0.34192486852407455,
+      "rewards/accuracy_reward": 0.2500000037252903,
+      "rewards/format_reward": 0.305555553175509,
+      "step": 75
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2547.402801513672,
+      "epoch": 0.13024850042844902,
+      "grad_norm": 0.12682628631591797,
+      "kl": 0.0037593841552734375,
+      "learning_rate": 9.976647061112284e-07,
+      "loss": 0.0589,
+      "reward": 1.3055555522441864,
+      "reward_std": 0.5543116554617882,
+      "rewards/accuracy_reward": 0.3472222276031971,
+      "rewards/format_reward": 0.6111111119389534,
+      "step": 76
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3289.8056030273438,
+      "epoch": 0.1319622964867181,
+      "grad_norm": 0.07940449565649033,
+      "kl": 0.00164031982421875,
+      "learning_rate": 9.973821595693026e-07,
+      "loss": 0.0351,
+      "reward": 0.7777777910232544,
+      "reward_std": 0.5557935982942581,
+      "rewards/accuracy_reward": 0.2361111156642437,
+      "rewards/format_reward": 0.3055555634200573,
+      "step": 77
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3333.75,
+      "epoch": 0.13367609254498714,
+      "grad_norm": 0.1280481368303299,
+      "kl": 0.00507354736328125,
+      "learning_rate": 9.970835319725696e-07,
+      "loss": -0.0008,
+      "reward": 0.27777778450399637,
+      "reward_std": 0.42627324536442757,
+      "rewards/accuracy_reward": 0.06944444589316845,
+      "rewards/format_reward": 0.13888889271765947,
+      "step": 78
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2680.7361450195312,
+      "epoch": 0.1353898886032562,
+      "grad_norm": 0.25829118490219116,
+      "kl": 0.0030956268310546875,
+      "learning_rate": 9.967688340551327e-07,
+      "loss": 0.1441,
+      "reward": 0.9999999925494194,
+      "reward_std": 0.46356454864144325,
+      "rewards/accuracy_reward": 0.2500000027939677,
+      "rewards/format_reward": 0.5,
+      "step": 79
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3091.388916015625,
+      "epoch": 0.13710368466152528,
+      "grad_norm": 0.2998567819595337,
+      "kl": 0.0024929046630859375,
+      "learning_rate": 9.96438077128741e-07,
+      "loss": 0.0411,
+      "reward": 0.5555555690079927,
+      "reward_std": 0.6594638489186764,
+      "rewards/accuracy_reward": 0.12500000186264515,
+      "rewards/format_reward": 0.3055555624887347,
+      "step": 80
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3257.0416870117188,
+      "epoch": 0.13881748071979436,
+      "grad_norm": 0.06869164109230042,
+      "kl": 0.0032901763916015625,
+      "learning_rate": 9.960912730823802e-07,
+      "loss": 0.0225,
+      "reward": 0.3888888927176595,
+      "reward_std": 0.1994824968278408,
+      "rewards/accuracy_reward": 0.11111111380159855,
+      "rewards/format_reward": 0.16666667070239782,
+      "step": 81
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3114.875,
+      "epoch": 0.1405312767780634,
+      "grad_norm": 0.246004119515419,
+      "kl": 0.0038585662841796875,
+      "learning_rate": 9.95728434381847e-07,
+      "loss": 0.1002,
+      "reward": 0.6527777910232544,
+      "reward_std": 0.4963437169790268,
+      "rewards/accuracy_reward": 0.1944444477558136,
+      "rewards/format_reward": 0.2638888955116272,
+      "step": 82
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2820.8612060546875,
+      "epoch": 0.14224507283633248,
+      "grad_norm": 0.1362665593624115,
+      "kl": 0.0025300979614257812,
+      "learning_rate": 9.953495740692994e-07,
+      "loss": 0.0321,
+      "reward": 1.152777798473835,
+      "reward_std": 0.4870201013982296,
+      "rewards/accuracy_reward": 0.3194444552063942,
+      "rewards/format_reward": 0.5138888955116272,
+      "step": 83
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3047.2361450195312,
+      "epoch": 0.14395886889460155,
+      "grad_norm": 0.12096895277500153,
+      "kl": 0.0046234130859375,
+      "learning_rate": 9.949547057627897e-07,
+      "loss": 0.06,
+      "reward": 0.7361111156642437,
+      "reward_std": 0.4963437244296074,
+      "rewards/accuracy_reward": 0.1944444514811039,
+      "rewards/format_reward": 0.3472222266718745,
+      "step": 84
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3108.5833740234375,
+      "epoch": 0.1456726649528706,
+      "grad_norm": 0.10810334980487823,
+      "kl": 0.0030975341796875,
+      "learning_rate": 9.945438436557734e-07,
+      "loss": 0.037,
+      "reward": 0.8472222238779068,
+      "reward_std": 0.4963437356054783,
+      "rewards/accuracy_reward": 0.2500000074505806,
+      "rewards/format_reward": 0.3472222238779068,
+      "step": 85
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2940.791717529297,
+      "epoch": 0.14738646101113967,
+      "grad_norm": 0.19223813712596893,
+      "kl": 0.011386871337890625,
+      "learning_rate": 9.941170025166e-07,
+      "loss": 0.0879,
+      "reward": 0.7638889048248529,
+      "reward_std": 0.48003676161170006,
+      "rewards/accuracy_reward": 0.20833333395421505,
+      "rewards/format_reward": 0.3472222276031971,
+      "step": 86
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3504.5138549804688,
+      "epoch": 0.14910025706940874,
+      "grad_norm": 0.058835774660110474,
+      "kl": 0.0025482177734375,
+      "learning_rate": 9.93674197687982e-07,
+      "loss": 0.006,
+      "reward": 0.4166666679084301,
+      "reward_std": 0.3332236036658287,
+      "rewards/accuracy_reward": 0.13888889271765947,
+      "rewards/format_reward": 0.13888889271765947,
+      "step": 87
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3452.3472290039062,
+      "epoch": 0.15081405312767782,
+      "grad_norm": 0.10622096061706543,
+      "kl": 0.00501251220703125,
+      "learning_rate": 9.932154450864423e-07,
+      "loss": 0.0373,
+      "reward": 0.36111112125217915,
+      "reward_std": 0.30703413113951683,
+      "rewards/accuracy_reward": 0.1111111119389534,
+      "rewards/format_reward": 0.13888888992369175,
+      "step": 88
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2943.8472290039062,
+      "epoch": 0.15252784918594686,
+      "grad_norm": 0.24842512607574463,
+      "kl": 0.00630950927734375,
+      "learning_rate": 9.927407612017446e-07,
+      "loss": 0.1236,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.36548174917697906,
+      "rewards/accuracy_reward": 0.06944444589316845,
+      "rewards/format_reward": 0.3194444440305233,
+      "step": 89
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3144.0416870117188,
+      "epoch": 0.15424164524421594,
+      "grad_norm": 0.08785022795200348,
+      "kl": 0.0038299560546875,
+      "learning_rate": 9.92250163096298e-07,
+      "loss": 0.0214,
+      "reward": 0.7916666716337204,
+      "reward_std": 0.29556556046009064,
+      "rewards/accuracy_reward": 0.2500000074505806,
+      "rewards/format_reward": 0.2916666753590107,
+      "step": 90
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3134.5833740234375,
+      "epoch": 0.155955441302485,
+      "grad_norm": 0.12506471574306488,
+      "kl": 0.007457733154296875,
+      "learning_rate": 9.91743668404545e-07,
+      "loss": 0.0601,
+      "reward": 0.6944444645196199,
+      "reward_std": 0.4941713698208332,
+      "rewards/accuracy_reward": 0.18055555783212185,
+      "rewards/format_reward": 0.3333333386108279,
+      "step": 91
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2990.5972290039062,
+      "epoch": 0.15766923736075408,
+      "grad_norm": 0.09640727192163467,
+      "kl": 0.004512786865234375,
+      "learning_rate": 9.912212953323279e-07,
+      "loss": -0.0125,
+      "reward": 0.8611110979691148,
+      "reward_std": 0.4442981481552124,
+      "rewards/accuracy_reward": 0.25000000186264515,
+      "rewards/format_reward": 0.36111112032085657,
+      "step": 92
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2980.263916015625,
+      "epoch": 0.15938303341902313,
+      "grad_norm": 0.1393478512763977,
+      "kl": 0.00595855712890625,
+      "learning_rate": 9.906830626562331e-07,
+      "loss": 0.0749,
+      "reward": 1.1250000149011612,
+      "reward_std": 0.7443297058343887,
+      "rewards/accuracy_reward": 0.33333333767950535,
+      "rewards/format_reward": 0.4583333395421505,
+      "step": 93
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3000.0694580078125,
+      "epoch": 0.1610968294772922,
+      "grad_norm": 0.1266450732946396,
+      "kl": 0.003871917724609375,
+      "learning_rate": 9.90128989722918e-07,
+      "loss": 0.0799,
+      "reward": 0.7916666753590107,
+      "reward_std": 0.40012458711862564,
+      "rewards/accuracy_reward": 0.22222222853451967,
+      "rewards/format_reward": 0.34722223319113255,
+      "step": 94
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3058.1112060546875,
+      "epoch": 0.16281062553556128,
+      "grad_norm": 0.14600639045238495,
+      "kl": 0.005794525146484375,
+      "learning_rate": 9.89559096448414e-07,
+      "loss": 0.0668,
+      "reward": 0.5972222313284874,
+      "reward_std": 0.3742276132106781,
+      "rewards/accuracy_reward": 0.15277778450399637,
+      "rewards/format_reward": 0.2916666753590107,
+      "step": 95
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3196.2361450195312,
+      "epoch": 0.16452442159383032,
+      "grad_norm": 0.22430367767810822,
+      "kl": 0.003925323486328125,
+      "learning_rate": 9.889734033174114e-07,
+      "loss": 0.0566,
+      "reward": 1.263888880610466,
+      "reward_std": 0.7597223073244095,
+      "rewards/accuracy_reward": 0.3888888880610466,
+      "rewards/format_reward": 0.486111119389534,
+      "step": 96
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3478.7916870117188,
+      "epoch": 0.1662382176520994,
+      "grad_norm": 0.11037839204072952,
+      "kl": 0.005950927734375,
+      "learning_rate": 9.883719313825227e-07,
+      "loss": 0.0342,
+      "reward": 0.583333345130086,
+      "reward_std": 0.6006828360259533,
+      "rewards/accuracy_reward": 0.1666666716337204,
+      "rewards/format_reward": 0.25000000558793545,
+      "step": 97
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3026.5277709960938,
+      "epoch": 0.16795201371036847,
+      "grad_norm": 0.1222737729549408,
+      "kl": 0.00817108154296875,
+      "learning_rate": 9.877547022635267e-07,
+      "loss": 0.042,
+      "reward": 1.0555555894970894,
+      "reward_std": 0.5561864748597145,
+      "rewards/accuracy_reward": 0.3055555559694767,
+      "rewards/format_reward": 0.4444444589316845,
+      "step": 98
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3361.638916015625,
+      "epoch": 0.16966580976863754,
+      "grad_norm": 0.1510419100522995,
+      "kl": 0.00934600830078125,
+      "learning_rate": 9.871217381465902e-07,
+      "loss": 0.0842,
+      "reward": 0.4444444617256522,
+      "reward_std": 0.3906443119049072,
+      "rewards/accuracy_reward": 0.1111111119389534,
+      "rewards/format_reward": 0.22222222294658422,
+      "step": 99
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3250.8194580078125,
+      "epoch": 0.1713796058269066,
+      "grad_norm": 0.15683910250663757,
+      "kl": 0.0079803466796875,
+      "learning_rate": 9.864730617834712e-07,
+      "loss": 0.0871,
+      "reward": 0.722222225740552,
+      "reward_std": 0.5869148373603821,
+      "rewards/accuracy_reward": 0.19444444589316845,
+      "rewards/format_reward": 0.33333334140479565,
+      "step": 100
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3126.6806030273438,
+      "epoch": 0.17309340188517566,
+      "grad_norm": 0.0631570890545845,
+      "kl": 0.005035400390625,
+      "learning_rate": 9.85808696490701e-07,
+      "loss": 0.0344,
+      "reward": 0.5555555559694767,
+      "reward_std": 0.04303314909338951,
+      "rewards/accuracy_reward": 0.1666666716337204,
+      "rewards/format_reward": 0.2222222276031971,
+      "step": 101
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3082.9166870117188,
+      "epoch": 0.17480719794344474,
+      "grad_norm": 0.0892498642206192,
+      "kl": 0.00846099853515625,
+      "learning_rate": 9.851286661487463e-07,
+      "loss": 0.0089,
+      "reward": 0.8194444477558136,
+      "reward_std": 0.5157952532172203,
+      "rewards/accuracy_reward": 0.1944444477558136,
+      "rewards/format_reward": 0.4305555522441864,
+      "step": 102
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3149.3333129882812,
+      "epoch": 0.17652099400171378,
+      "grad_norm": 0.1086692363023758,
+      "kl": 0.00589752197265625,
+      "learning_rate": 9.844329952011504e-07,
+      "loss": 0.0502,
+      "reward": 0.7777777798473835,
+      "reward_std": 0.2971716746687889,
+      "rewards/accuracy_reward": 0.2083333395421505,
+      "rewards/format_reward": 0.3611111156642437,
+      "step": 103
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3506.9722290039062,
+      "epoch": 0.17823479005998286,
+      "grad_norm": 0.10262292623519897,
+      "kl": 0.005126953125,
+      "learning_rate": 9.837217086536547e-07,
+      "loss": 0.0261,
+      "reward": 0.4583333320915699,
+      "reward_std": 0.6020341292023659,
+      "rewards/accuracy_reward": 0.13888889364898205,
+      "rewards/format_reward": 0.18055556155741215,
+      "step": 104
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2822.2222290039062,
+      "epoch": 0.17994858611825193,
+      "grad_norm": 0.11509492248296738,
+      "kl": 0.00792694091796875,
+      "learning_rate": 9.829948320733e-07,
+      "loss": 0.0501,
+      "reward": 0.8333333283662796,
+      "reward_std": 0.6226537488400936,
+      "rewards/accuracy_reward": 0.18055556062608957,
+      "rewards/format_reward": 0.4722222238779068,
+      "step": 105
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3389.6527709960938,
+      "epoch": 0.181662382176521,
+      "grad_norm": 0.11453983187675476,
+      "kl": 0.00846099853515625,
+      "learning_rate": 9.822523915875077e-07,
+      "loss": 0.0623,
+      "reward": 0.6666666679084301,
+      "reward_std": 0.7060182616114616,
+      "rewards/accuracy_reward": 0.20833334047347307,
+      "rewards/format_reward": 0.25000000838190317,
+      "step": 106
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3259.6112060546875,
+      "epoch": 0.18337617823479005,
+      "grad_norm": 0.09651084989309311,
+      "kl": 0.007171630859375,
+      "learning_rate": 9.8149441388314e-07,
+      "loss": 0.0375,
+      "reward": 0.5833333441987634,
+      "reward_std": 0.3762567453086376,
+      "rewards/accuracy_reward": 0.16666667349636555,
+      "rewards/format_reward": 0.2500000027939677,
+      "step": 107
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3319.1944580078125,
+      "epoch": 0.18508997429305912,
+      "grad_norm": 0.11137856543064117,
+      "kl": 0.00550079345703125,
+      "learning_rate": 9.807209262055415e-07,
+      "loss": 0.03,
+      "reward": 0.736111112870276,
+      "reward_std": 0.6152903288602829,
+      "rewards/accuracy_reward": 0.19444444589316845,
+      "rewards/format_reward": 0.3472222210839391,
+      "step": 108
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3011.9443969726562,
+      "epoch": 0.1868037703513282,
+      "grad_norm": 0.1661546230316162,
+      "kl": 0.00968170166015625,
+      "learning_rate": 9.799319563575593e-07,
+      "loss": 0.0582,
+      "reward": 0.5555555559694767,
+      "reward_std": 0.40824829787015915,
+      "rewards/accuracy_reward": 0.11111111380159855,
+      "rewards/format_reward": 0.33333333022892475,
+      "step": 109
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3077.3472290039062,
+      "epoch": 0.18851756640959727,
+      "grad_norm": 0.1285841166973114,
+      "kl": 0.0084228515625,
+      "learning_rate": 9.791275326985434e-07,
+      "loss": 0.0404,
+      "reward": 0.7638888955116272,
+      "reward_std": 0.8195342868566513,
+      "rewards/accuracy_reward": 0.19444445241242647,
+      "rewards/format_reward": 0.3750000037252903,
+      "step": 110
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2989.02783203125,
+      "epoch": 0.19023136246786632,
+      "grad_norm": 0.14638738334178925,
+      "kl": 0.00787353515625,
+      "learning_rate": 9.783076841433279e-07,
+      "loss": 0.0678,
+      "reward": 0.6805555634200573,
+      "reward_std": 0.540618360042572,
+      "rewards/accuracy_reward": 0.15277778171002865,
+      "rewards/format_reward": 0.3750000037252903,
+      "step": 111
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3064.2222290039062,
+      "epoch": 0.1919451585261354,
+      "grad_norm": 0.11260731518268585,
+      "kl": 0.0069580078125,
+      "learning_rate": 9.774724401611918e-07,
+      "loss": 0.0528,
+      "reward": 0.624999993480742,
+      "reward_std": 0.45954934880137444,
+      "rewards/accuracy_reward": 0.12500000186264515,
+      "rewards/format_reward": 0.37500000838190317,
+      "step": 112
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3019.5138549804688,
+      "epoch": 0.19365895458440446,
+      "grad_norm": 0.1483815312385559,
+      "kl": 0.0084686279296875,
+      "learning_rate": 9.76621830774799e-07,
+      "loss": 0.0892,
+      "reward": 1.1250000149011612,
+      "reward_std": 0.7124863266944885,
+      "rewards/accuracy_reward": 0.3194444477558136,
+      "rewards/format_reward": 0.4861111119389534,
+      "step": 113
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2765.2083435058594,
+      "epoch": 0.1953727506426735,
+      "grad_norm": 0.16946516931056976,
+      "kl": 0.00933837890625,
+      "learning_rate": 9.757558865591196e-07,
+      "loss": 0.0612,
+      "reward": 0.8611110933125019,
+      "reward_std": 0.5443289652466774,
+      "rewards/accuracy_reward": 0.22222222853451967,
+      "rewards/format_reward": 0.41666667349636555,
+      "step": 114
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2844.77783203125,
+      "epoch": 0.19708654670094258,
+      "grad_norm": 0.3654565215110779,
+      "kl": 0.00984954833984375,
+      "learning_rate": 9.748746386403305e-07,
+      "loss": 0.2113,
+      "reward": 0.875,
+      "reward_std": 0.7314287945628166,
+      "rewards/accuracy_reward": 0.2361111156642437,
+      "rewards/format_reward": 0.4027777910232544,
+      "step": 115
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3334.25,
+      "epoch": 0.19880034275921166,
+      "grad_norm": 0.1866183876991272,
+      "kl": 0.01085662841796875,
+      "learning_rate": 9.739781186946978e-07,
+      "loss": 0.0509,
+      "reward": 0.5138888889923692,
+      "reward_std": 0.3636069521307945,
+      "rewards/accuracy_reward": 0.1111111156642437,
+      "rewards/format_reward": 0.2916666688397527,
+      "step": 116
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3231.9722290039062,
+      "epoch": 0.20051413881748073,
+      "grad_norm": 0.1388145536184311,
+      "kl": 0.01290130615234375,
+      "learning_rate": 9.730663589474364e-07,
+      "loss": 0.0397,
+      "reward": 0.9027777630835772,
+      "reward_std": 0.766443993896246,
+      "rewards/accuracy_reward": 0.2500000037252903,
+      "rewards/format_reward": 0.40277778543531895,
+      "step": 117
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2966.15283203125,
+      "epoch": 0.20222793487574978,
+      "grad_norm": 0.10939842462539673,
+      "kl": 0.011932373046875,
+      "learning_rate": 9.721393921715533e-07,
+      "loss": 0.0173,
+      "reward": 0.6527777835726738,
+      "reward_std": 0.4645882025361061,
+      "rewards/accuracy_reward": 0.12500000279396772,
+      "rewards/format_reward": 0.4027777835726738,
+      "step": 118
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2932.3333740234375,
+      "epoch": 0.20394173093401885,
+      "grad_norm": 0.10793702304363251,
+      "kl": 0.01213836669921875,
+      "learning_rate": 9.711972516866678e-07,
+      "loss": 0.0573,
+      "reward": 0.861111119389534,
+      "reward_std": 0.21574701368808746,
+      "rewards/accuracy_reward": 0.236111119389534,
+      "rewards/format_reward": 0.3888888955116272,
+      "step": 119
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3096.666748046875,
+      "epoch": 0.20565552699228792,
+      "grad_norm": 0.22288325428962708,
+      "kl": 0.01317596435546875,
+      "learning_rate": 9.70239971357816e-07,
+      "loss": 0.1225,
+      "reward": 0.8750000149011612,
+      "reward_std": 0.8644967824220657,
+      "rewards/accuracy_reward": 0.26388889644294977,
+      "rewards/format_reward": 0.3472222276031971,
+      "step": 120
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3175.625,
+      "epoch": 0.207369323050557,
+      "grad_norm": 0.11317700147628784,
+      "kl": 0.0115814208984375,
+      "learning_rate": 9.692675855942318e-07,
+      "loss": 0.0349,
+      "reward": 0.8611111342906952,
+      "reward_std": 0.6463145054876804,
+      "rewards/accuracy_reward": 0.2361111156642437,
+      "rewards/format_reward": 0.38888888992369175,
+      "step": 121
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3247.8333740234375,
+      "epoch": 0.20908311910882604,
+      "grad_norm": 0.18068967759609222,
+      "kl": 0.014404296875,
+      "learning_rate": 9.682801293481108e-07,
+      "loss": 0.0904,
+      "reward": 0.4305555671453476,
+      "reward_std": 0.5079665929079056,
+      "rewards/accuracy_reward": 0.041666666977107525,
+      "rewards/format_reward": 0.3472222276031971,
+      "step": 122
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2834.2222595214844,
+      "epoch": 0.21079691516709512,
+      "grad_norm": 0.15124721825122833,
+      "kl": 0.012481689453125,
+      "learning_rate": 9.67277638113354e-07,
+      "loss": 0.0635,
+      "reward": 1.2222222611308098,
+      "reward_std": 0.6941533647477627,
+      "rewards/accuracy_reward": 0.37500000186264515,
+      "rewards/format_reward": 0.47222221828997135,
+      "step": 123
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3222.3472900390625,
+      "epoch": 0.2125107112253642,
+      "grad_norm": 0.14290107786655426,
+      "kl": 0.012481689453125,
+      "learning_rate": 9.662601479242914e-07,
+      "loss": 0.0404,
+      "reward": 0.8611111268401146,
+      "reward_std": 0.5691947191953659,
+      "rewards/accuracy_reward": 0.25000000558793545,
+      "rewards/format_reward": 0.3611111119389534,
+      "step": 124
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3088.8333740234375,
+      "epoch": 0.21422450728363324,
+      "grad_norm": 0.08310513198375702,
+      "kl": 0.0166168212890625,
+      "learning_rate": 9.652276953543877e-07,
+      "loss": 0.0133,
+      "reward": 0.8055555820465088,
+      "reward_std": 0.3680921755731106,
+      "rewards/accuracy_reward": 0.2500000037252903,
+      "rewards/format_reward": 0.3055555559694767,
+      "step": 125
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3026.0555419921875,
+      "epoch": 0.2159383033419023,
+      "grad_norm": 0.20979027450084686,
+      "kl": 0.017181396484375,
+      "learning_rate": 9.641803175149264e-07,
+      "loss": 0.076,
+      "reward": 0.8611111156642437,
+      "reward_std": 0.41928989440202713,
+      "rewards/accuracy_reward": 0.22222222108393908,
+      "rewards/format_reward": 0.416666672565043,
+      "step": 126
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3460.875,
+      "epoch": 0.21765209940017138,
+      "grad_norm": 0.15380728244781494,
+      "kl": 0.0160064697265625,
+      "learning_rate": 9.631180520536777e-07,
+      "loss": 0.0459,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.5213519968092442,
+      "rewards/accuracy_reward": 0.06944444496184587,
+      "rewards/format_reward": 0.15277777891606092,
+      "step": 127
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3339.1806030273438,
+      "epoch": 0.21936589545844046,
+      "grad_norm": 0.0945175364613533,
+      "kl": 0.0178375244140625,
+      "learning_rate": 9.62040937153543e-07,
+      "loss": 0.0209,
+      "reward": 0.6527777910232544,
+      "reward_std": 0.46317092329263687,
+      "rewards/accuracy_reward": 0.2083333395421505,
+      "rewards/format_reward": 0.2361111156642437,
+      "step": 128
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2740.3195190429688,
+      "epoch": 0.2210796915167095,
+      "grad_norm": 0.15982748568058014,
+      "kl": 0.01837158203125,
+      "learning_rate": 9.60949011531184e-07,
+      "loss": 0.0037,
+      "reward": 0.8750000149011612,
+      "reward_std": 0.5248102210462093,
+      "rewards/accuracy_reward": 0.2222222238779068,
+      "rewards/format_reward": 0.4305555559694767,
+      "step": 129
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2644.3472900390625,
+      "epoch": 0.22279348757497858,
+      "grad_norm": 0.10350772738456726,
+      "kl": 0.0185546875,
+      "learning_rate": 9.598423144356312e-07,
+      "loss": 0.0416,
+      "reward": 1.5833333153277636,
+      "reward_std": 0.4627453200519085,
+      "rewards/accuracy_reward": 0.4861111082136631,
+      "rewards/format_reward": 0.6111111212521791,
+      "step": 130
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3329.263916015625,
+      "epoch": 0.22450728363324765,
+      "grad_norm": 0.17227336764335632,
+      "kl": 0.0158233642578125,
+      "learning_rate": 9.587208856468713e-07,
+      "loss": 0.0527,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.4853021502494812,
+      "rewards/accuracy_reward": 0.11111111473292112,
+      "rewards/format_reward": 0.23611111380159855,
+      "step": 131
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3231.4861450195312,
+      "epoch": 0.2262210796915167,
+      "grad_norm": 0.13778822124004364,
+      "kl": 0.016204833984375,
+      "learning_rate": 9.575847654744196e-07,
+      "loss": 0.0544,
+      "reward": 0.611111119389534,
+      "reward_std": 0.541582465171814,
+      "rewards/accuracy_reward": 0.16666667070239782,
+      "rewards/format_reward": 0.2777777872979641,
+      "step": 132
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3083.2222900390625,
+      "epoch": 0.22793487574978577,
+      "grad_norm": 0.12422922253608704,
+      "kl": 0.0192108154296875,
+      "learning_rate": 9.564339947558697e-07,
+      "loss": 0.017,
+      "reward": 1.0694444477558136,
+      "reward_std": 0.5992536917328835,
+      "rewards/accuracy_reward": 0.31944444961845875,
+      "rewards/format_reward": 0.430555559694767,
+      "step": 133
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2589.6806030273438,
+      "epoch": 0.22964867180805484,
+      "grad_norm": 0.24903494119644165,
+      "kl": 0.01654052734375,
+      "learning_rate": 9.552686148554252e-07,
+      "loss": 0.1379,
+      "reward": 1.3055555671453476,
+      "reward_std": 0.8013041242957115,
+      "rewards/accuracy_reward": 0.3611111165955663,
+      "rewards/format_reward": 0.5833333469927311,
+      "step": 134
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2682.638916015625,
+      "epoch": 0.23136246786632392,
+      "grad_norm": 0.16912828385829926,
+      "kl": 0.0197601318359375,
+      "learning_rate": 9.540886676624145e-07,
+      "loss": 0.0699,
+      "reward": 1.1805555671453476,
+      "reward_std": 0.5904464609920979,
+      "rewards/accuracy_reward": 0.30555556435137987,
+      "rewards/format_reward": 0.5694444477558136,
+      "step": 135
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3100.4444580078125,
+      "epoch": 0.23307626392459296,
+      "grad_norm": 0.14741510152816772,
+      "kl": 0.0220794677734375,
+      "learning_rate": 9.528941955897839e-07,
+      "loss": 0.0595,
+      "reward": 0.8750000232830644,
+      "reward_std": 0.6136412769556046,
+      "rewards/accuracy_reward": 0.2638888917863369,
+      "rewards/format_reward": 0.3472222285345197,
+      "step": 136
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3276.1805419921875,
+      "epoch": 0.23479005998286204,
+      "grad_norm": 0.12443236261606216,
+      "kl": 0.020843505859375,
+      "learning_rate": 9.516852415725732e-07,
+      "loss": 0.0334,
+      "reward": 1.1944444477558136,
+      "reward_std": 0.5957729890942574,
+      "rewards/accuracy_reward": 0.361111119389534,
+      "rewards/format_reward": 0.4722222238779068,
+      "step": 137
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3190.138916015625,
+      "epoch": 0.2365038560411311,
+      "grad_norm": 0.12088010460138321,
+      "kl": 0.0197601318359375,
+      "learning_rate": 9.504618490663726e-07,
+      "loss": 0.0239,
+      "reward": 0.9305555606260896,
+      "reward_std": 0.39455174282193184,
+      "rewards/accuracy_reward": 0.2777777910232544,
+      "rewards/format_reward": 0.3750000009313226,
+      "step": 138
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3037.0833129882812,
+      "epoch": 0.23821765209940018,
+      "grad_norm": 0.17049150168895721,
+      "kl": 0.02288818359375,
+      "learning_rate": 9.492240620457606e-07,
+      "loss": 0.075,
+      "reward": 0.6111111156642437,
+      "reward_std": 0.35632818564772606,
+      "rewards/accuracy_reward": 0.13888888992369175,
+      "rewards/format_reward": 0.3333333358168602,
+      "step": 139
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2627.3611450195312,
+      "epoch": 0.23993144815766923,
+      "grad_norm": 0.14227531850337982,
+      "kl": 0.028778076171875,
+      "learning_rate": 9.479719250027239e-07,
+      "loss": 0.0286,
+      "reward": 0.9027777910232544,
+      "reward_std": 0.2613905444741249,
+      "rewards/accuracy_reward": 0.18055556155741215,
+      "rewards/format_reward": 0.541666679084301,
+      "step": 140
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3192.625,
+      "epoch": 0.2416452442159383,
+      "grad_norm": 0.16459161043167114,
+      "kl": 0.02606201171875,
+      "learning_rate": 9.467054829450571e-07,
+      "loss": 0.0413,
+      "reward": 0.7500000149011612,
+      "reward_std": 0.4791666865348816,
+      "rewards/accuracy_reward": 0.19444444496184587,
+      "rewards/format_reward": 0.361111119389534,
+      "step": 141
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3326.263916015625,
+      "epoch": 0.24335904027420738,
+      "grad_norm": 0.08248669654130936,
+      "kl": 0.0257568359375,
+      "learning_rate": 9.454247813947455e-07,
+      "loss": -0.0021,
+      "reward": 0.486111119389534,
+      "reward_std": 0.24812544882297516,
+      "rewards/accuracy_reward": 0.1527777835726738,
+      "rewards/format_reward": 0.180555559694767,
+      "step": 142
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3274.7500610351562,
+      "epoch": 0.24507283633247642,
+      "grad_norm": 0.22874736785888672,
+      "kl": 0.027557373046875,
+      "learning_rate": 9.441298663863289e-07,
+      "loss": 0.0603,
+      "reward": 0.6944444477558136,
+      "reward_std": 0.5954620316624641,
+      "rewards/accuracy_reward": 0.16666666511446238,
+      "rewards/format_reward": 0.361111119389534,
+      "step": 143
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3433.0416870117188,
+      "epoch": 0.2467866323907455,
+      "grad_norm": 0.1357790231704712,
+      "kl": 0.025115966796875,
+      "learning_rate": 9.428207844652466e-07,
+      "loss": 0.0337,
+      "reward": 0.611111119389534,
+      "reward_std": 0.7063797116279602,
+      "rewards/accuracy_reward": 0.180555559694767,
+      "rewards/format_reward": 0.2500000074505806,
+      "step": 144
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3047.5972900390625,
+      "epoch": 0.24850042844901457,
+      "grad_norm": 0.10790549218654633,
+      "kl": 0.02679443359375,
+      "learning_rate": 9.414975826861651e-07,
+      "loss": -0.0015,
+      "reward": 0.8194444347172976,
+      "reward_std": 0.45097024738788605,
+      "rewards/accuracy_reward": 0.2222222276031971,
+      "rewards/format_reward": 0.37500000931322575,
+      "step": 145
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2496.180633544922,
+      "epoch": 0.25021422450728364,
+      "grad_norm": 0.2421943098306656,
+      "kl": 0.025360107421875,
+      "learning_rate": 9.401603086112854e-07,
+      "loss": 0.0966,
+      "reward": 1.1666667014360428,
+      "reward_std": 0.7269039005041122,
+      "rewards/accuracy_reward": 0.3194444514811039,
+      "rewards/format_reward": 0.5277777835726738,
+      "step": 146
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3213.861083984375,
+      "epoch": 0.2519280205655527,
+      "grad_norm": 0.07842709124088287,
+      "kl": 0.029083251953125,
+      "learning_rate": 9.388090103086343e-07,
+      "loss": 0.0114,
+      "reward": 0.5833333358168602,
+      "reward_std": 0.14478403329849243,
+      "rewards/accuracy_reward": 0.1388888917863369,
+      "rewards/format_reward": 0.3055555671453476,
+      "step": 147
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3094.3611450195312,
+      "epoch": 0.2536418166238218,
+      "grad_norm": 0.08900830894708633,
+      "kl": 0.02301025390625,
+      "learning_rate": 9.374437363503368e-07,
+      "loss": 0.0566,
+      "reward": 0.5138888917863369,
+      "reward_std": 0.17996380850672722,
+      "rewards/accuracy_reward": 0.125,
+      "rewards/format_reward": 0.2638888917863369,
+      "step": 148
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2830.7361450195312,
+      "epoch": 0.25535561268209084,
+      "grad_norm": 0.5457747578620911,
+      "kl": 0.05902099609375,
+      "learning_rate": 9.360645358108695e-07,
+      "loss": 0.0158,
+      "reward": 1.0694444477558136,
+      "reward_std": 0.6218058541417122,
+      "rewards/accuracy_reward": 0.2916666716337204,
+      "rewards/format_reward": 0.486111119389534,
+      "step": 149
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2843.3333740234375,
+      "epoch": 0.2570694087403599,
+      "grad_norm": 0.3755730986595154,
+      "kl": 0.02838134765625,
+      "learning_rate": 9.34671458265297e-07,
+      "loss": 0.1583,
+      "reward": 1.2083333432674408,
+      "reward_std": 0.6026154272258282,
+      "rewards/accuracy_reward": 0.36111111380159855,
+      "rewards/format_reward": 0.486111119389534,
+      "step": 150
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3075.763916015625,
+      "epoch": 0.258783204798629,
+      "grad_norm": 0.418891578912735,
+      "kl": 0.0416259765625,
+      "learning_rate": 9.332645537874899e-07,
+      "loss": 0.0858,
+      "reward": 0.5277777947485447,
+      "reward_std": 0.4114787243306637,
+      "rewards/accuracy_reward": 0.09722222574055195,
+      "rewards/format_reward": 0.3333333432674408,
+      "step": 151
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3111.8056030273438,
+      "epoch": 0.26049700085689803,
+      "grad_norm": 0.24780075252056122,
+      "kl": 0.03948974609375,
+      "learning_rate": 9.318438729483249e-07,
+      "loss": 0.0987,
+      "reward": 0.6527777761220932,
+      "reward_std": 0.5695527195930481,
+      "rewards/accuracy_reward": 0.1944444477558136,
+      "rewards/format_reward": 0.2638888992369175,
+      "step": 152
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3033.763916015625,
+      "epoch": 0.2622107969151671,
+      "grad_norm": 0.31493860483169556,
+      "kl": 0.036956787109375,
+      "learning_rate": 9.304094668138669e-07,
+      "loss": 0.0483,
+      "reward": 0.7777777798473835,
+      "reward_std": 0.7054093405604362,
+      "rewards/accuracy_reward": 0.180555559694767,
+      "rewards/format_reward": 0.4166666828095913,
+      "step": 153
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2864.388916015625,
+      "epoch": 0.2639245929734362,
+      "grad_norm": 0.17504319548606873,
+      "kl": 0.03851318359375,
+      "learning_rate": 9.289613869435336e-07,
+      "loss": 0.0353,
+      "reward": 0.7083333432674408,
+      "reward_std": 0.5801292359828949,
+      "rewards/accuracy_reward": 0.19444444682449102,
+      "rewards/format_reward": 0.31944444961845875,
+      "step": 154
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3274.7222290039062,
+      "epoch": 0.2656383890317052,
+      "grad_norm": 0.130802720785141,
+      "kl": 0.04754638671875,
+      "learning_rate": 9.274996853882425e-07,
+      "loss": 0.0159,
+      "reward": 0.8472222350537777,
+      "reward_std": 0.5149499326944351,
+      "rewards/accuracy_reward": 0.25000000186264515,
+      "rewards/format_reward": 0.3472222248092294,
+      "step": 155
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2833.4444580078125,
+      "epoch": 0.26735218508997427,
+      "grad_norm": 0.24871142208576202,
+      "kl": 0.04052734375,
+      "learning_rate": 9.260244146885391e-07,
+      "loss": 0.0251,
+      "reward": 0.9861111305654049,
+      "reward_std": 0.5550614818930626,
+      "rewards/accuracy_reward": 0.2500000046566129,
+      "rewards/format_reward": 0.486111112870276,
+      "step": 156
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2903.638885498047,
+      "epoch": 0.26906598114824337,
+      "grad_norm": 0.158583402633667,
+      "kl": 0.04901123046875,
+      "learning_rate": 9.245356278727093e-07,
+      "loss": -0.0172,
+      "reward": 1.0555555894970894,
+      "reward_std": 0.6695602312684059,
+      "rewards/accuracy_reward": 0.30555555783212185,
+      "rewards/format_reward": 0.4444444514811039,
+      "step": 157
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2771.236114501953,
+      "epoch": 0.2707797772065124,
+      "grad_norm": 0.2626900374889374,
+      "kl": 0.051513671875,
+      "learning_rate": 9.230333784548726e-07,
+      "loss": 0.0563,
+      "reward": 0.8055555671453476,
+      "reward_std": 0.6322761699557304,
+      "rewards/accuracy_reward": 0.1805555559694767,
+      "rewards/format_reward": 0.4444444477558136,
+      "step": 158
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2995.0694580078125,
+      "epoch": 0.27249357326478146,
+      "grad_norm": 0.7491080164909363,
+      "kl": 0.06500244140625,
+      "learning_rate": 9.215177204330587e-07,
+      "loss": 0.1298,
+      "reward": 0.847222201526165,
+      "reward_std": 0.8498382568359375,
+      "rewards/accuracy_reward": 0.22222222667187452,
+      "rewards/format_reward": 0.4027777835726738,
+      "step": 159
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2974.9166870117188,
+      "epoch": 0.27420736932305056,
+      "grad_norm": 0.2567845284938812,
+      "kl": 0.06201171875,
+      "learning_rate": 9.199887082872672e-07,
+      "loss": 0.0524,
+      "reward": 0.9583333283662796,
+      "reward_std": 0.5167308412492275,
+      "rewards/accuracy_reward": 0.30555556155741215,
+      "rewards/format_reward": 0.3472222285345197,
+      "step": 160
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2585.77783203125,
+      "epoch": 0.2759211653813196,
+      "grad_norm": 0.2760615050792694,
+      "kl": 0.0694580078125,
+      "learning_rate": 9.184463969775083e-07,
+      "loss": 0.0275,
+      "reward": 0.9166666567325592,
+      "reward_std": 0.587900884449482,
+      "rewards/accuracy_reward": 0.30555556155741215,
+      "rewards/format_reward": 0.30555556155741215,
+      "step": 161
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2197.25,
+      "epoch": 0.2776349614395887,
+      "grad_norm": 0.5792744755744934,
+      "kl": 0.0704345703125,
+      "learning_rate": 9.168908419418278e-07,
+      "loss": 0.0347,
+      "reward": 0.611111119389534,
+      "reward_std": 0.4948512986302376,
+      "rewards/accuracy_reward": 0.15277778077870607,
+      "rewards/format_reward": 0.3055555671453476,
+      "step": 162
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2838.041748046875,
+      "epoch": 0.27934875749785776,
+      "grad_norm": 0.39986100792884827,
+      "kl": 0.0880126953125,
+      "learning_rate": 9.153220990943145e-07,
+      "loss": 0.0897,
+      "reward": 0.625,
+      "reward_std": 0.6566977351903915,
+      "rewards/accuracy_reward": 0.1944444477558136,
+      "rewards/format_reward": 0.23611111287027597,
+      "step": 163
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2120.2916870117188,
+      "epoch": 0.2810625535561268,
+      "grad_norm": 0.5093821287155151,
+      "kl": 0.0892333984375,
+      "learning_rate": 9.137402248230903e-07,
+      "loss": 0.0789,
+      "reward": 0.7777777835726738,
+      "reward_std": 0.4542044475674629,
+      "rewards/accuracy_reward": 0.236111119389534,
+      "rewards/format_reward": 0.305555559694767,
+      "step": 164
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2339.6666870117188,
+      "epoch": 0.2827763496143959,
+      "grad_norm": 0.3379420340061188,
+      "kl": 0.095947265625,
+      "learning_rate": 9.121452759882831e-07,
+      "loss": -0.01,
+      "reward": 1.0555555745959282,
+      "reward_std": 0.7722257673740387,
+      "rewards/accuracy_reward": 0.3472222285345197,
+      "rewards/format_reward": 0.3611111119389534,
+      "step": 165
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2644.361114501953,
+      "epoch": 0.28449014567266495,
+      "grad_norm": 0.4091556668281555,
+      "kl": 0.105224609375,
+      "learning_rate": 9.105373099199835e-07,
+      "loss": 0.0473,
+      "reward": 0.9166666716337204,
+      "reward_std": 0.8636802136898041,
+      "rewards/accuracy_reward": 0.2777777807787061,
+      "rewards/format_reward": 0.361111112870276,
+      "step": 166
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2401.9722595214844,
+      "epoch": 0.286203941730934,
+      "grad_norm": 0.6180943250656128,
+      "kl": 0.1217041015625,
+      "learning_rate": 9.08916384416183e-07,
+      "loss": 0.1078,
+      "reward": 1.0138888955116272,
+      "reward_std": 1.0287161767482758,
+      "rewards/accuracy_reward": 0.3888888955116272,
+      "rewards/format_reward": 0.23611111659556627,
+      "step": 167
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2471.9584045410156,
+      "epoch": 0.2879177377892031,
+      "grad_norm": 0.3897995352745056,
+      "kl": 0.119384765625,
+      "learning_rate": 9.072825577406981e-07,
+      "loss": 0.0262,
+      "reward": 1.0555555522441864,
+      "reward_std": 0.5160078145563602,
+      "rewards/accuracy_reward": 0.3333333432674408,
+      "rewards/format_reward": 0.3888888917863369,
+      "step": 168
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2073.916717529297,
+      "epoch": 0.28963153384747214,
+      "grad_norm": 0.74821537733078,
+      "kl": 0.1395263671875,
+      "learning_rate": 9.056358886210747e-07,
+      "loss": 0.0509,
+      "reward": 0.902777798473835,
+      "reward_std": 0.7328798174858093,
+      "rewards/accuracy_reward": 0.2916666688397527,
+      "rewards/format_reward": 0.3194444514811039,
+      "step": 169
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1999.2499694824219,
+      "epoch": 0.2913453299057412,
+      "grad_norm": 0.5790112018585205,
+      "kl": 0.1368408203125,
+      "learning_rate": 9.039764362464775e-07,
+      "loss": -0.0285,
+      "reward": 0.9444444701075554,
+      "reward_std": 0.6172067150473595,
+      "rewards/accuracy_reward": 0.34722222946584225,
+      "rewards/format_reward": 0.2500000074505806,
+      "step": 170
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2040.3472595214844,
+      "epoch": 0.2930591259640103,
+      "grad_norm": 0.6991499066352844,
+      "kl": 0.13037109375,
+      "learning_rate": 9.023042602655623e-07,
+      "loss": -0.0061,
+      "reward": 0.8750000186264515,
+      "reward_std": 0.5527190193533897,
+      "rewards/accuracy_reward": 0.2361111119389534,
+      "rewards/format_reward": 0.40277778916060925,
+      "step": 171
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2682.625,
+      "epoch": 0.29477292202227934,
+      "grad_norm": 0.4274706244468689,
+      "kl": 0.1610107421875,
+      "learning_rate": 9.00619420784333e-07,
+      "loss": 0.0389,
+      "reward": 0.8888888955116272,
+      "reward_std": 0.9098298698663712,
+      "rewards/accuracy_reward": 0.26388889737427235,
+      "rewards/format_reward": 0.361111119389534,
+      "step": 172
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2140.3333435058594,
+      "epoch": 0.29648671808054844,
+      "grad_norm": 1.2440991401672363,
+      "kl": 0.156494140625,
+      "learning_rate": 8.989219783639795e-07,
+      "loss": 0.1369,
+      "reward": 0.7916666865348816,
+      "reward_std": 0.7525217533111572,
+      "rewards/accuracy_reward": 0.1666666679084301,
+      "rewards/format_reward": 0.4583333395421505,
+      "step": 173
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2162.361083984375,
+      "epoch": 0.2982005141388175,
+      "grad_norm": 2.159301519393921,
+      "kl": 0.1800537109375,
+      "learning_rate": 8.972119940187017e-07,
+      "loss": 0.1231,
+      "reward": 0.6666666716337204,
+      "reward_std": 0.6304600611329079,
+      "rewards/accuracy_reward": 0.13888889364898205,
+      "rewards/format_reward": 0.3888888955116272,
+      "step": 174
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2967.8333740234375,
+      "epoch": 0.29991431019708653,
+      "grad_norm": 0.6146534085273743,
+      "kl": 0.29296875,
+      "learning_rate": 8.95489529213517e-07,
+      "loss": 0.0622,
+      "reward": 0.4027777872979641,
+      "reward_std": 0.4733867570757866,
+      "rewards/accuracy_reward": 0.0555555559694767,
+      "rewards/format_reward": 0.2916666753590107,
+      "step": 175
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2475.7777709960938,
+      "epoch": 0.30162810625535563,
+      "grad_norm": 0.9705228209495544,
+      "kl": 0.2490234375,
+      "learning_rate": 8.93754645862049e-07,
+      "loss": 0.0784,
+      "reward": 0.8750000149011612,
+      "reward_std": 0.6140558868646622,
+      "rewards/accuracy_reward": 0.1805555559694767,
+      "rewards/format_reward": 0.5138888880610466,
+      "step": 176
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3104.0139770507812,
+      "epoch": 0.3033419023136247,
+      "grad_norm": 0.41929054260253906,
+      "kl": 0.32421875,
+      "learning_rate": 8.920074063243045e-07,
+      "loss": 0.0399,
+      "reward": 0.5277777845039964,
+      "reward_std": 0.3559510372579098,
+      "rewards/accuracy_reward": 0.11111111287027597,
+      "rewards/format_reward": 0.30555556435137987,
+      "step": 177
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2869.6527709960938,
+      "epoch": 0.3050556983718937,
+      "grad_norm": 0.4340916574001312,
+      "kl": 0.351318359375,
+      "learning_rate": 8.902478734044297e-07,
+      "loss": 0.0451,
+      "reward": 0.8750000074505806,
+      "reward_std": 0.37828588113188744,
+      "rewards/accuracy_reward": 0.2222222238779068,
+      "rewards/format_reward": 0.430555559694767,
+      "step": 178
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3403.9444580078125,
+      "epoch": 0.3067694944301628,
+      "grad_norm": 0.5245348811149597,
+      "kl": 0.4765625,
+      "learning_rate": 8.884761103484547e-07,
+      "loss": 0.0707,
+      "reward": 0.5416666753590107,
+      "reward_std": 0.47885870188474655,
+      "rewards/accuracy_reward": 0.15277778450399637,
+      "rewards/format_reward": 0.23611111380159855,
+      "step": 179
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2249.7083740234375,
+      "epoch": 0.30848329048843187,
+      "grad_norm": 1.083481788635254,
+      "kl": 0.35009765625,
+      "learning_rate": 8.866921808420184e-07,
+      "loss": 0.1105,
+      "reward": 1.4305555820465088,
+      "reward_std": 0.4940013214945793,
+      "rewards/accuracy_reward": 0.3333333395421505,
+      "rewards/format_reward": 0.763888880610466,
+      "step": 180
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3113.90283203125,
+      "epoch": 0.3101970865467009,
+      "grad_norm": 0.5710152983665466,
+      "kl": 0.525390625,
+      "learning_rate": 8.848961490080805e-07,
+      "loss": 0.0798,
+      "reward": 0.7916666716337204,
+      "reward_std": 0.4283023402094841,
+      "rewards/accuracy_reward": 0.2222222238779068,
+      "rewards/format_reward": 0.3472222313284874,
+      "step": 181
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3370.9166870117188,
+      "epoch": 0.31191088260497,
+      "grad_norm": 0.744907557964325,
+      "kl": 0.662109375,
+      "learning_rate": 8.830880794046162e-07,
+      "loss": 0.0642,
+      "reward": 0.22222222853451967,
+      "reward_std": 0.27216554805636406,
+      "rewards/accuracy_reward": 0.013888888992369175,
+      "rewards/format_reward": 0.19444444961845875,
+      "step": 182
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3109.4861450195312,
+      "epoch": 0.31362467866323906,
+      "grad_norm": 0.9323793649673462,
+      "kl": 0.5927734375,
+      "learning_rate": 8.81268037022296e-07,
+      "loss": 0.0847,
+      "reward": 0.5972222201526165,
+      "reward_std": 0.518895335495472,
+      "rewards/accuracy_reward": 0.12500000279396772,
+      "rewards/format_reward": 0.34722223225980997,
+      "step": 183
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3036.0833129882812,
+      "epoch": 0.31533847472150817,
+      "grad_norm": 0.7612030506134033,
+      "kl": 0.666015625,
+      "learning_rate": 8.794360872821486e-07,
+      "loss": 0.0732,
+      "reward": 0.7777777686715126,
+      "reward_std": 0.781570628285408,
+      "rewards/accuracy_reward": 0.15277778171002865,
+      "rewards/format_reward": 0.47222222574055195,
+      "step": 184
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3103.138916015625,
+      "epoch": 0.3170522707797772,
+      "grad_norm": 0.8818280100822449,
+      "kl": 0.7119140625,
+      "learning_rate": 8.775922960332108e-07,
+      "loss": 0.0613,
+      "reward": 0.6388889141380787,
+      "reward_std": 0.4765222370624542,
+      "rewards/accuracy_reward": 0.13888888992369175,
+      "rewards/format_reward": 0.3611111156642437,
+      "step": 185
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2773.90283203125,
+      "epoch": 0.31876606683804626,
+      "grad_norm": 0.6143203377723694,
+      "kl": 0.6201171875,
+      "learning_rate": 8.757367295501594e-07,
+      "loss": 0.0686,
+      "reward": 0.791666679084301,
+      "reward_std": 0.4058704674243927,
+      "rewards/accuracy_reward": 0.1666666679084301,
+      "rewards/format_reward": 0.4583333358168602,
+      "step": 186
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2821.4583740234375,
+      "epoch": 0.32047986289631536,
+      "grad_norm": 2.1052632331848145,
+      "kl": 0.5546875,
+      "learning_rate": 8.738694545309298e-07,
+      "loss": 0.1962,
+      "reward": 0.7638888955116272,
+      "reward_std": 0.47831880301237106,
+      "rewards/accuracy_reward": 0.16666667349636555,
+      "rewards/format_reward": 0.4305555634200573,
+      "step": 187
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3268.40283203125,
+      "epoch": 0.3221936589545844,
+      "grad_norm": 0.7343023419380188,
+      "kl": 0.6640625,
+      "learning_rate": 8.719905380943182e-07,
+      "loss": 0.0963,
+      "reward": 0.3888888955116272,
+      "reward_std": 0.5786792561411858,
+      "rewards/accuracy_reward": 0.0555555559694767,
+      "rewards/format_reward": 0.2777777872979641,
+      "step": 188
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2894.0972900390625,
+      "epoch": 0.32390745501285345,
+      "grad_norm": 1.0675281286239624,
+      "kl": 0.6748046875,
+      "learning_rate": 8.701000477775687e-07,
+      "loss": 0.1042,
+      "reward": 0.4444444626569748,
+      "reward_std": 0.4907895401120186,
+      "rewards/accuracy_reward": 0.013888888992369175,
+      "rewards/format_reward": 0.416666679084301,
+      "step": 189
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3079.3056030273438,
+      "epoch": 0.32562125107112255,
+      "grad_norm": 0.7137811183929443,
+      "kl": 0.6826171875,
+      "learning_rate": 8.681980515339463e-07,
+      "loss": 0.0989,
+      "reward": 0.541666679084301,
+      "reward_std": 0.5043292306363583,
+      "rewards/accuracy_reward": 0.06944444589316845,
+      "rewards/format_reward": 0.4027777835726738,
+      "step": 190
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3071.763916015625,
+      "epoch": 0.3273350471293916,
+      "grad_norm": 0.7186347842216492,
+      "kl": 0.6953125,
+      "learning_rate": 8.662846177302938e-07,
+      "loss": 0.0886,
+      "reward": 0.5833333358168602,
+      "reward_std": 0.5841793119907379,
+      "rewards/accuracy_reward": 0.08333333488553762,
+      "rewards/format_reward": 0.4166666679084301,
+      "step": 191
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2750.3194580078125,
+      "epoch": 0.32904884318766064,
+      "grad_norm": 0.8925949931144714,
+      "kl": 0.6806640625,
+      "learning_rate": 8.643598151445749e-07,
+      "loss": 0.1256,
+      "reward": 1.0694444328546524,
+      "reward_std": 0.6757483929395676,
+      "rewards/accuracy_reward": 0.20833334047347307,
+      "rewards/format_reward": 0.6527777910232544,
+      "step": 192
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2392.611083984375,
+      "epoch": 0.33076263924592975,
+      "grad_norm": 0.5500184297561646,
+      "kl": 0.59814453125,
+      "learning_rate": 8.624237129634014e-07,
+      "loss": 0.0753,
+      "reward": 0.9861111082136631,
+      "reward_std": 0.6858525797724724,
+      "rewards/accuracy_reward": 0.2083333395421505,
+      "rewards/format_reward": 0.5694444365799427,
+      "step": 193
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2629.9305419921875,
+      "epoch": 0.3324764353041988,
+      "grad_norm": 1.4036595821380615,
+      "kl": 0.8115234375,
+      "learning_rate": 8.604763807795471e-07,
+      "loss": 0.0248,
+      "reward": 1.222222238779068,
+      "reward_std": 0.7275054007768631,
+      "rewards/accuracy_reward": 0.2500000046566129,
+      "rewards/format_reward": 0.722222238779068,
+      "step": 194
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2822.6527709960938,
+      "epoch": 0.3341902313624679,
+      "grad_norm": 0.9383707642555237,
+      "kl": 0.7744140625,
+      "learning_rate": 8.58517888589445e-07,
+      "loss": 0.0842,
+      "reward": 0.9583333358168602,
+      "reward_std": 0.6252798363566399,
+      "rewards/accuracy_reward": 0.16666666977107525,
+      "rewards/format_reward": 0.6250000037252903,
+      "step": 195
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2831.65283203125,
+      "epoch": 0.33590402742073694,
+      "grad_norm": 0.9345480799674988,
+      "kl": 0.7021484375,
+      "learning_rate": 8.56548306790673e-07,
+      "loss": 0.1095,
+      "reward": 0.9444444626569748,
+      "reward_std": 0.6248819530010223,
+      "rewards/accuracy_reward": 0.13888888899236917,
+      "rewards/format_reward": 0.6666666716337204,
+      "step": 196
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2750.7777709960938,
+      "epoch": 0.337617823479006,
+      "grad_norm": 0.6670540571212769,
+      "kl": 0.685546875,
+      "learning_rate": 8.54567706179422e-07,
+      "loss": 0.1075,
+      "reward": 0.9861111119389534,
+      "reward_std": 0.5248102322220802,
+      "rewards/accuracy_reward": 0.180555559694767,
+      "rewards/format_reward": 0.6250000074505806,
+      "step": 197
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2877.9861450195312,
+      "epoch": 0.3393316195372751,
+      "grad_norm": 0.6243178844451904,
+      "kl": 0.7509765625,
+      "learning_rate": 8.525761579479519e-07,
+      "loss": 0.0843,
+      "reward": 1.0555555671453476,
+      "reward_std": 0.4111253023147583,
+      "rewards/accuracy_reward": 0.19444444868713617,
+      "rewards/format_reward": 0.6666666865348816,
+      "step": 198
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2674.138916015625,
+      "epoch": 0.34104541559554413,
+      "grad_norm": 1.4835889339447021,
+      "kl": 0.7109375,
+      "learning_rate": 8.505737336820326e-07,
+      "loss": 0.1377,
+      "reward": 0.875,
+      "reward_std": 0.4817770943045616,
+      "rewards/accuracy_reward": 0.1111111156642437,
+      "rewards/format_reward": 0.6527777761220932,
+      "step": 199
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2610.1666870117188,
+      "epoch": 0.3427592116538132,
+      "grad_norm": 1.3861608505249023,
+      "kl": 0.7060546875,
+      "learning_rate": 8.485605053583704e-07,
+      "loss": 0.1008,
+      "reward": 0.986111119389534,
+      "reward_std": 0.6297199055552483,
+      "rewards/accuracy_reward": 0.15277777798473835,
+      "rewards/format_reward": 0.6805555671453476,
+      "step": 200
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2719.513916015625,
+      "epoch": 0.3444730077120823,
+      "grad_norm": 1.214066982269287,
+      "kl": 0.755859375,
+      "learning_rate": 8.465365453420214e-07,
+      "loss": 0.0994,
+      "reward": 1.097222238779068,
+      "reward_std": 0.511199101805687,
+      "rewards/accuracy_reward": 0.1388888917863369,
+      "rewards/format_reward": 0.8194444477558136,
+      "step": 201
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2794.7500610351562,
+      "epoch": 0.3461868037703513,
+      "grad_norm": 0.9259114861488342,
+      "kl": 0.85546875,
+      "learning_rate": 8.445019263837897e-07,
+      "loss": 0.0963,
+      "reward": 1.2083333730697632,
+      "reward_std": 0.45388972014188766,
+      "rewards/accuracy_reward": 0.2500000009313226,
+      "rewards/format_reward": 0.7083333432674408,
+      "step": 202
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2657.65283203125,
+      "epoch": 0.34790059982862037,
+      "grad_norm": 0.9675770998001099,
+      "kl": 0.9267578125,
+      "learning_rate": 8.42456721617613e-07,
+      "loss": 0.1232,
+      "reward": 1.2500000149011612,
+      "reward_std": 0.8653819561004639,
+      "rewards/accuracy_reward": 0.23611111659556627,
+      "rewards/format_reward": 0.7777777761220932,
+      "step": 203
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2665.486083984375,
+      "epoch": 0.3496143958868895,
+      "grad_norm": 1.2522883415222168,
+      "kl": 0.87109375,
+      "learning_rate": 8.404010045579339e-07,
+      "loss": 0.0604,
+      "reward": 0.8888888955116272,
+      "reward_std": 0.5926736369729042,
+      "rewards/accuracy_reward": 0.09722222574055195,
+      "rewards/format_reward": 0.6944444626569748,
+      "step": 204
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2659.9722290039062,
+      "epoch": 0.3513281919451585,
+      "grad_norm": 1.2460371255874634,
+      "kl": 0.9208984375,
+      "learning_rate": 8.383348490970566e-07,
+      "loss": 0.143,
+      "reward": 1.3611111491918564,
+      "reward_std": 0.7807096689939499,
+      "rewards/accuracy_reward": 0.26388889644294977,
+      "rewards/format_reward": 0.8333333283662796,
+      "step": 205
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2711.4445190429688,
+      "epoch": 0.35304198800342756,
+      "grad_norm": 1.2466338872909546,
+      "kl": 0.9326171875,
+      "learning_rate": 8.362583295024916e-07,
+      "loss": 0.0888,
+      "reward": 0.8888888955116272,
+      "reward_std": 0.4819314032793045,
+      "rewards/accuracy_reward": 0.055555556900799274,
+      "rewards/format_reward": 0.7777777761220932,
+      "step": 206
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2571.138916015625,
+      "epoch": 0.35475578406169667,
+      "grad_norm": 0.7316084504127502,
+      "kl": 0.8232421875,
+      "learning_rate": 8.341715204142854e-07,
+      "loss": 0.0954,
+      "reward": 1.1527777761220932,
+      "reward_std": 0.5541091933846474,
+      "rewards/accuracy_reward": 0.22222223225980997,
+      "rewards/format_reward": 0.7083333432674408,
+      "step": 207
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2744.5556030273438,
+      "epoch": 0.3564695801199657,
+      "grad_norm": 0.9308451414108276,
+      "kl": 0.8291015625,
+      "learning_rate": 8.320744968423391e-07,
+      "loss": 0.0961,
+      "reward": 1.1666667014360428,
+      "reward_std": 0.4660206064581871,
+      "rewards/accuracy_reward": 0.15277777798473835,
+      "rewards/format_reward": 0.861111119389534,
+      "step": 208
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2710.3055419921875,
+      "epoch": 0.3581833761782348,
+      "grad_norm": 0.861219584941864,
+      "kl": 0.826171875,
+      "learning_rate": 8.299673341637108e-07,
+      "loss": 0.0934,
+      "reward": 0.9722222238779068,
+      "reward_std": 0.5195175558328629,
+      "rewards/accuracy_reward": 0.06944444589316845,
+      "rewards/format_reward": 0.8333333134651184,
+      "step": 209
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2682.2361450195312,
+      "epoch": 0.35989717223650386,
+      "grad_norm": 0.9901695251464844,
+      "kl": 0.705078125,
+      "learning_rate": 8.278501081199061e-07,
+      "loss": 0.0675,
+      "reward": 1.0277778059244156,
+      "reward_std": 0.5829172991216183,
+      "rewards/accuracy_reward": 0.09722222294658422,
+      "rewards/format_reward": 0.833333358168602,
+      "step": 210
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3082.5833129882812,
+      "epoch": 0.3616109682947729,
+      "grad_norm": 0.9780495762825012,
+      "kl": 0.740234375,
+      "learning_rate": 8.257228948141567e-07,
+      "loss": 0.0635,
+      "reward": 0.8750000149011612,
+      "reward_std": 0.4285326674580574,
+      "rewards/accuracy_reward": 0.041666666977107525,
+      "rewards/format_reward": 0.7916666716337204,
+      "step": 211
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2808.0,
+      "epoch": 0.363324764353042,
+      "grad_norm": 0.7250691652297974,
+      "kl": 0.6376953125,
+      "learning_rate": 8.23585770708684e-07,
+      "loss": 0.0783,
+      "reward": 1.4861111342906952,
+      "reward_std": 0.5571966022253036,
+      "rewards/accuracy_reward": 0.3472222350537777,
+      "rewards/format_reward": 0.7916666567325592,
+      "step": 212
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3058.8611450195312,
+      "epoch": 0.36503856041131105,
+      "grad_norm": 0.9827658534049988,
+      "kl": 0.619140625,
+      "learning_rate": 8.214388126219512e-07,
+      "loss": 0.0416,
+      "reward": 0.9583333432674408,
+      "reward_std": 0.6335423514246941,
+      "rewards/accuracy_reward": 0.11111111380159855,
+      "rewards/format_reward": 0.7361111044883728,
+      "step": 213
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3149.3333740234375,
+      "epoch": 0.3667523564695801,
+      "grad_norm": 0.5304873585700989,
+      "kl": 0.59521484375,
+      "learning_rate": 8.192820977259012e-07,
+      "loss": 0.0755,
+      "reward": 0.8472222238779068,
+      "reward_std": 0.5018017217516899,
+      "rewards/accuracy_reward": 0.09722222574055195,
+      "rewards/format_reward": 0.6527777761220932,
+      "step": 214
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3244.77783203125,
+      "epoch": 0.3684661525278492,
+      "grad_norm": 0.5707263350486755,
+      "kl": 0.53076171875,
+      "learning_rate": 8.17115703543184e-07,
+      "loss": 0.0701,
+      "reward": 1.1388888955116272,
+      "reward_std": 0.9169216901063919,
+      "rewards/accuracy_reward": 0.23611112032085657,
+      "rewards/format_reward": 0.6666666641831398,
+      "step": 215
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3028.6528930664062,
+      "epoch": 0.37017994858611825,
+      "grad_norm": 0.5431216359138489,
+      "kl": 0.4140625,
+      "learning_rate": 8.149397079443693e-07,
+      "loss": 0.0184,
+      "reward": 1.0,
+      "reward_std": 0.5379247963428497,
+      "rewards/accuracy_reward": 0.1666666716337204,
+      "rewards/format_reward": 0.6666666716337204,
+      "step": 216
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2745.5972900390625,
+      "epoch": 0.3718937446443873,
+      "grad_norm": 0.5233212113380432,
+      "kl": 0.376953125,
+      "learning_rate": 8.127541891451473e-07,
+      "loss": 0.0288,
+      "reward": 1.1388889104127884,
+      "reward_std": 0.37009186670184135,
+      "rewards/accuracy_reward": 0.19444444868713617,
+      "rewards/format_reward": 0.7500000149011612,
+      "step": 217
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2928.5972290039062,
+      "epoch": 0.3736075407026564,
+      "grad_norm": 0.640915036201477,
+      "kl": 0.35791015625,
+      "learning_rate": 8.105592257035178e-07,
+      "loss": 0.0105,
+      "reward": 1.2083333134651184,
+      "reward_std": 0.3840879425406456,
+      "rewards/accuracy_reward": 0.23611111752688885,
+      "rewards/format_reward": 0.736111119389534,
+      "step": 218
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3421.0972290039062,
+      "epoch": 0.37532133676092544,
+      "grad_norm": 0.39103758335113525,
+      "kl": 0.4453125,
+      "learning_rate": 8.083548965169663e-07,
+      "loss": 0.0357,
+      "reward": 0.666666679084301,
+      "reward_std": 0.63420694693923,
+      "rewards/accuracy_reward": 0.06944444496184587,
+      "rewards/format_reward": 0.5277777910232544,
+      "step": 219
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3180.263916015625,
+      "epoch": 0.37703513281919454,
+      "grad_norm": 0.5637526512145996,
+      "kl": 0.385009765625,
+      "learning_rate": 8.061412808196279e-07,
+      "loss": 0.0577,
+      "reward": 0.7916666939854622,
+      "reward_std": 0.5208309143781662,
+      "rewards/accuracy_reward": 0.0972222238779068,
+      "rewards/format_reward": 0.5972222313284874,
+      "step": 220
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3347.8750610351562,
+      "epoch": 0.3787489288774636,
+      "grad_norm": 0.6664404273033142,
+      "kl": 0.38037109375,
+      "learning_rate": 8.039184581794389e-07,
+      "loss": 0.0478,
+      "reward": 0.6527777761220932,
+      "reward_std": 0.6700074002146721,
+      "rewards/accuracy_reward": 0.08333333395421505,
+      "rewards/format_reward": 0.486111119389534,
+      "step": 221
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2943.40283203125,
+      "epoch": 0.38046272493573263,
+      "grad_norm": 0.4021422863006592,
+      "kl": 0.302490234375,
+      "learning_rate": 8.016865084952783e-07,
+      "loss": 0.0219,
+      "reward": 1.2083333358168602,
+      "reward_std": 0.29920290410518646,
+      "rewards/accuracy_reward": 0.2500000074505806,
+      "rewards/format_reward": 0.7083333358168602,
+      "step": 222
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3259.875,
+      "epoch": 0.38217652099400173,
+      "grad_norm": 0.33430805802345276,
+      "kl": 0.34228515625,
+      "learning_rate": 7.994455119940934e-07,
+      "loss": 0.0304,
+      "reward": 0.6527777872979641,
+      "reward_std": 0.27941545471549034,
+      "rewards/accuracy_reward": 0.0833333358168602,
+      "rewards/format_reward": 0.4861111156642437,
+      "step": 223
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3342.0694580078125,
+      "epoch": 0.3838903170522708,
+      "grad_norm": 0.6777502298355103,
+      "kl": 0.40234375,
+      "learning_rate": 7.971955492280181e-07,
+      "loss": 0.0796,
+      "reward": 0.5277777835726738,
+      "reward_std": 0.5462377443909645,
+      "rewards/accuracy_reward": 0.041666666977107525,
+      "rewards/format_reward": 0.4444444552063942,
+      "step": 224
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3149.8194580078125,
+      "epoch": 0.3856041131105398,
+      "grad_norm": 0.5160468220710754,
+      "kl": 0.340087890625,
+      "learning_rate": 7.949367010714766e-07,
+      "loss": 0.0212,
+      "reward": 1.2500000149011612,
+      "reward_std": 0.7483058720827103,
+      "rewards/accuracy_reward": 0.2500000046566129,
+      "rewards/format_reward": 0.75,
+      "step": 225
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3071.7777709960938,
+      "epoch": 0.3873179091688089,
+      "grad_norm": 0.4626597762107849,
+      "kl": 0.4052734375,
+      "learning_rate": 7.926690487182766e-07,
+      "loss": 0.016,
+      "reward": 0.8194444552063942,
+      "reward_std": 0.5300310403108597,
+      "rewards/accuracy_reward": 0.06944444589316845,
+      "rewards/format_reward": 0.6805555447936058,
+      "step": 226
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3198.5277709960938,
+      "epoch": 0.389031705227078,
+      "grad_norm": 0.5830844640731812,
+      "kl": 0.3916015625,
+      "learning_rate": 7.903926736786907e-07,
+      "loss": 0.0462,
+      "reward": 0.8611111268401146,
+      "reward_std": 0.7832111120223999,
+      "rewards/accuracy_reward": 0.15277778264135122,
+      "rewards/format_reward": 0.5555555559694767,
+      "step": 227
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3048.5556030273438,
+      "epoch": 0.390745501285347,
+      "grad_norm": 0.43366798758506775,
+      "kl": 0.3828125,
+      "learning_rate": 7.881076577765265e-07,
+      "loss": 0.024,
+      "reward": 1.3472222238779068,
+      "reward_std": 0.5785843208432198,
+      "rewards/accuracy_reward": 0.3333333292976022,
+      "rewards/format_reward": 0.6805555522441864,
+      "step": 228
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3198.2777709960938,
+      "epoch": 0.3924592973436161,
+      "grad_norm": 0.721635103225708,
+      "kl": 0.421875,
+      "learning_rate": 7.858140831461858e-07,
+      "loss": 0.0741,
+      "reward": 1.0694444626569748,
+      "reward_std": 0.6419415846467018,
+      "rewards/accuracy_reward": 0.2222222276031971,
+      "rewards/format_reward": 0.625,
+      "step": 229
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3199.9306030273438,
+      "epoch": 0.39417309340188517,
+      "grad_norm": 0.5975983738899231,
+      "kl": 0.50439453125,
+      "learning_rate": 7.835120322297115e-07,
+      "loss": 0.0397,
+      "reward": 1.125,
+      "reward_std": 0.6707755327224731,
+      "rewards/accuracy_reward": 0.2500000037252903,
+      "rewards/format_reward": 0.625,
+      "step": 230
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3298.90283203125,
+      "epoch": 0.39588688946015427,
+      "grad_norm": 0.5230339169502258,
+      "kl": 0.44140625,
+      "learning_rate": 7.812015877738252e-07,
+      "loss": 0.0553,
+      "reward": 1.0972222238779068,
+      "reward_std": 0.7510210201144218,
+      "rewards/accuracy_reward": 0.2500000074505806,
+      "rewards/format_reward": 0.5972222238779068,
+      "step": 231
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2978.625,
+      "epoch": 0.3976006855184233,
+      "grad_norm": 0.4864640533924103,
+      "kl": 0.42626953125,
+      "learning_rate": 7.788828328269524e-07,
+      "loss": 0.04,
+      "reward": 1.3888889104127884,
+      "reward_std": 0.4389140121638775,
+      "rewards/accuracy_reward": 0.3194444552063942,
+      "rewards/format_reward": 0.75,
+      "step": 232
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2690.7083129882812,
+      "epoch": 0.39931448157669236,
+      "grad_norm": 0.41488051414489746,
+      "kl": 0.330078125,
+      "learning_rate": 7.765558507362374e-07,
+      "loss": 0.0505,
+      "reward": 1.6111111342906952,
+      "reward_std": 0.6035833917558193,
+      "rewards/accuracy_reward": 0.3888888992369175,
+      "rewards/format_reward": 0.8333333432674408,
+      "step": 233
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3084.0833740234375,
+      "epoch": 0.40102827763496146,
+      "grad_norm": 0.9714931845664978,
+      "kl": 0.52490234375,
+      "learning_rate": 7.742207251445473e-07,
+      "loss": 0.0225,
+      "reward": 0.7083333320915699,
+      "reward_std": 0.3626026399433613,
+      "rewards/accuracy_reward": 0.11111111287027597,
+      "rewards/format_reward": 0.4861111231148243,
+      "step": 234
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3456.4722290039062,
+      "epoch": 0.4027420736932305,
+      "grad_norm": 0.6735730767250061,
+      "kl": 0.5576171875,
+      "learning_rate": 7.718775399874654e-07,
+      "loss": 0.0413,
+      "reward": 0.7638889029622078,
+      "reward_std": 0.6374682560563087,
+      "rewards/accuracy_reward": 0.12500000465661287,
+      "rewards/format_reward": 0.5138889029622078,
+      "step": 235
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3396.0277709960938,
+      "epoch": 0.40445586975149955,
+      "grad_norm": 0.46086516976356506,
+      "kl": 0.51416015625,
+      "learning_rate": 7.69526379490275e-07,
+      "loss": 0.0611,
+      "reward": 0.7777777761220932,
+      "reward_std": 0.7557945251464844,
+      "rewards/accuracy_reward": 0.12500000279396772,
+      "rewards/format_reward": 0.5277777910232544,
+      "step": 236
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2660.041717529297,
+      "epoch": 0.40616966580976865,
+      "grad_norm": 0.49919629096984863,
+      "kl": 0.375244140625,
+      "learning_rate": 7.671673281649303e-07,
+      "loss": 0.0153,
+      "reward": 0.9305555373430252,
+      "reward_std": 0.4616476893424988,
+      "rewards/accuracy_reward": 0.1388888917863369,
+      "rewards/format_reward": 0.6527777761220932,
+      "step": 237
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3412.5833740234375,
+      "epoch": 0.4078834618680377,
+      "grad_norm": 0.4641401171684265,
+      "kl": 0.44921875,
+      "learning_rate": 7.648004708070207e-07,
+      "loss": 0.0416,
+      "reward": 0.9166666716337204,
+      "reward_std": 0.6401529386639595,
+      "rewards/accuracy_reward": 0.15277778077870607,
+      "rewards/format_reward": 0.6111111044883728,
+      "step": 238
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3040.27783203125,
+      "epoch": 0.40959725792630675,
+      "grad_norm": 0.3767884373664856,
+      "kl": 0.4150390625,
+      "learning_rate": 7.624258924927209e-07,
+      "loss": 0.0663,
+      "reward": 1.0555555745959282,
+      "reward_std": 0.6340665742754936,
+      "rewards/accuracy_reward": 0.26388890016824007,
+      "rewards/format_reward": 0.5277777835726738,
+      "step": 239
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2810.4583740234375,
+      "epoch": 0.41131105398457585,
+      "grad_norm": 0.7160300612449646,
+      "kl": 0.27001953125,
+      "learning_rate": 7.600436785757339e-07,
+      "loss": 0.0574,
+      "reward": 1.3194444477558136,
+      "reward_std": 0.5911112986505032,
+      "rewards/accuracy_reward": 0.2916666753590107,
+      "rewards/format_reward": 0.736111119389534,
+      "step": 240
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2855.4444885253906,
+      "epoch": 0.4130248500428449,
+      "grad_norm": 0.6084269285202026,
+      "kl": 0.359375,
+      "learning_rate": 7.57653914684223e-07,
+      "loss": 0.0512,
+      "reward": 1.0000000447034836,
+      "reward_std": 0.5819622427225113,
+      "rewards/accuracy_reward": 0.25,
+      "rewards/format_reward": 0.5,
+      "step": 241
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3339.5694580078125,
+      "epoch": 0.414738646101114,
+      "grad_norm": 0.9977325201034546,
+      "kl": 0.34228515625,
+      "learning_rate": 7.552566867177336e-07,
+      "loss": 0.0482,
+      "reward": 1.0277778059244156,
+      "reward_std": 0.8119450323283672,
+      "rewards/accuracy_reward": 0.2500000037252903,
+      "rewards/format_reward": 0.5277777910232544,
+      "step": 242
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3133.416748046875,
+      "epoch": 0.41645244215938304,
+      "grad_norm": 0.47933000326156616,
+      "kl": 0.3671875,
+      "learning_rate": 7.528520808441057e-07,
+      "loss": 0.0389,
+      "reward": 1.0138888880610466,
+      "reward_std": 0.8036867156624794,
+      "rewards/accuracy_reward": 0.19444444496184587,
+      "rewards/format_reward": 0.6250000074505806,
+      "step": 243
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2973.013916015625,
+      "epoch": 0.4181662382176521,
+      "grad_norm": 0.7056116461753845,
+      "kl": 0.3857421875,
+      "learning_rate": 7.504401834963763e-07,
+      "loss": 0.0406,
+      "reward": 1.0416666716337204,
+      "reward_std": 0.8239623233675957,
+      "rewards/accuracy_reward": 0.23611111007630825,
+      "rewards/format_reward": 0.569444440305233,
+      "step": 244
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2305.291717529297,
+      "epoch": 0.4198800342759212,
+      "grad_norm": 0.8029626607894897,
+      "kl": 0.284912109375,
+      "learning_rate": 7.480210813696732e-07,
+      "loss": 0.072,
+      "reward": 1.5833333432674408,
+      "reward_std": 0.43726158887147903,
+      "rewards/accuracy_reward": 0.4027777789160609,
+      "rewards/format_reward": 0.7777777910232544,
+      "step": 245
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3021.0000610351562,
+      "epoch": 0.42159383033419023,
+      "grad_norm": 0.5367264151573181,
+      "kl": 0.51416015625,
+      "learning_rate": 7.455948614180983e-07,
+      "loss": 0.069,
+      "reward": 0.9305555447936058,
+      "reward_std": 0.6338964849710464,
+      "rewards/accuracy_reward": 0.25000000931322575,
+      "rewards/format_reward": 0.4305555559694767,
+      "step": 246
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3298.6250610351562,
+      "epoch": 0.4233076263924593,
+      "grad_norm": 0.7639601826667786,
+      "kl": 0.70458984375,
+      "learning_rate": 7.431616108516021e-07,
+      "loss": 0.0682,
+      "reward": 0.625,
+      "reward_std": 0.7283531203866005,
+      "rewards/accuracy_reward": 0.11111111287027597,
+      "rewards/format_reward": 0.4027777835726738,
+      "step": 247
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2943.1944580078125,
+      "epoch": 0.4250214224507284,
+      "grad_norm": 0.7967272400856018,
+      "kl": 0.626953125,
+      "learning_rate": 7.407214171328491e-07,
+      "loss": 0.0337,
+      "reward": 0.8055555671453476,
+      "reward_std": 0.5930216982960701,
+      "rewards/accuracy_reward": 0.1944444477558136,
+      "rewards/format_reward": 0.416666679084301,
+      "step": 248
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3075.5833740234375,
+      "epoch": 0.4267352185089974,
+      "grad_norm": 0.9339537024497986,
+      "kl": 0.60400390625,
+      "learning_rate": 7.382743679740741e-07,
+      "loss": 0.0485,
+      "reward": 0.6805555522441864,
+      "reward_std": 0.3991912603378296,
+      "rewards/accuracy_reward": 0.1250000037252903,
+      "rewards/format_reward": 0.4305555671453476,
+      "step": 249
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3042.9722900390625,
+      "epoch": 0.4284490145672665,
+      "grad_norm": 0.6018530130386353,
+      "kl": 0.580078125,
+      "learning_rate": 7.358205513339286e-07,
+      "loss": 0.0693,
+      "reward": 1.0138888955116272,
+      "reward_std": 0.9293834567070007,
+      "rewards/accuracy_reward": 0.2361111156642437,
+      "rewards/format_reward": 0.541666679084301,
+      "step": 250
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3095.1527709960938,
+      "epoch": 0.4301628106255356,
+      "grad_norm": 0.6325446367263794,
+      "kl": 0.607421875,
+      "learning_rate": 7.333600554143203e-07,
+      "loss": 0.1134,
+      "reward": 0.9166666865348816,
+      "reward_std": 0.7987345978617668,
+      "rewards/accuracy_reward": 0.20833333767950535,
+      "rewards/format_reward": 0.5000000074505806,
+      "step": 251
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3202.1111450195312,
+      "epoch": 0.4318766066838046,
+      "grad_norm": 0.7408186793327332,
+      "kl": 0.6162109375,
+      "learning_rate": 7.308929686572423e-07,
+      "loss": 0.0686,
+      "reward": 0.680555559694767,
+      "reward_std": 0.486115962266922,
+      "rewards/accuracy_reward": 0.12500000186264515,
+      "rewards/format_reward": 0.4305555634200573,
+      "step": 252
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3191.1250610351562,
+      "epoch": 0.43359040274207367,
+      "grad_norm": 0.5966067910194397,
+      "kl": 0.51708984375,
+      "learning_rate": 7.284193797415932e-07,
+      "loss": 0.0627,
+      "reward": 1.0555555671453476,
+      "reward_std": 0.8806970864534378,
+      "rewards/accuracy_reward": 0.2638888955116272,
+      "rewards/format_reward": 0.5277777910232544,
+      "step": 253
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3091.638916015625,
+      "epoch": 0.43530419880034277,
+      "grad_norm": 0.6850226521492004,
+      "kl": 0.4912109375,
+      "learning_rate": 7.25939377579991e-07,
+      "loss": 0.0783,
+      "reward": 0.9305555745959282,
+      "reward_std": 0.7012817077338696,
+      "rewards/accuracy_reward": 0.2361111156642437,
+      "rewards/format_reward": 0.4583333432674408,
+      "step": 254
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2927.9166870117188,
+      "epoch": 0.4370179948586118,
+      "grad_norm": 0.7290365099906921,
+      "kl": 0.47216796875,
+      "learning_rate": 7.234530513155761e-07,
+      "loss": 0.0949,
+      "reward": 0.9027777910232544,
+      "reward_std": 0.7394577413797379,
+      "rewards/accuracy_reward": 0.180555559694767,
+      "rewards/format_reward": 0.541666679084301,
+      "step": 255
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2891.5277709960938,
+      "epoch": 0.4387317909168809,
+      "grad_norm": 0.7196898460388184,
+      "kl": 0.38330078125,
+      "learning_rate": 7.209604903188073e-07,
+      "loss": 0.051,
+      "reward": 0.8888888955116272,
+      "reward_std": 0.6187723875045776,
+      "rewards/accuracy_reward": 0.15277778171002865,
+      "rewards/format_reward": 0.5833333283662796,
+      "step": 256
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2936.9583129882812,
+      "epoch": 0.44044558697514996,
+      "grad_norm": 0.7010894417762756,
+      "kl": 0.48193359375,
+      "learning_rate": 7.184617841842498e-07,
+      "loss": 0.1164,
+      "reward": 0.8888888955116272,
+      "reward_std": 0.7065365388989449,
+      "rewards/accuracy_reward": 0.1944444514811039,
+      "rewards/format_reward": 0.5,
+      "step": 257
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3047.013916015625,
+      "epoch": 0.442159383033419,
+      "grad_norm": 0.5052915811538696,
+      "kl": 0.4599609375,
+      "learning_rate": 7.159570227273543e-07,
+      "loss": 0.0301,
+      "reward": 0.7916666641831398,
+      "reward_std": 0.5421573370695114,
+      "rewards/accuracy_reward": 0.1388888917863369,
+      "rewards/format_reward": 0.5138888843357563,
+      "step": 258
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2858.2084350585938,
+      "epoch": 0.4438731790916881,
+      "grad_norm": 1.2074464559555054,
+      "kl": 0.42578125,
+      "learning_rate": 7.134462959812286e-07,
+      "loss": 0.1245,
+      "reward": 1.1388889104127884,
+      "reward_std": 0.7573273852467537,
+      "rewards/accuracy_reward": 0.2638888955116272,
+      "rewards/format_reward": 0.6111111044883728,
+      "step": 259
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2916.7222290039062,
+      "epoch": 0.44558697514995715,
+      "grad_norm": 0.4421754777431488,
+      "kl": 0.47509765625,
+      "learning_rate": 7.10929694193402e-07,
+      "loss": 0.0576,
+      "reward": 1.1388888955116272,
+      "reward_std": 0.7476281188428402,
+      "rewards/accuracy_reward": 0.2500000037252903,
+      "rewards/format_reward": 0.6388888955116272,
+      "step": 260
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2938.0972290039062,
+      "epoch": 0.4473007712082262,
+      "grad_norm": 0.6367775201797485,
+      "kl": 0.537109375,
+      "learning_rate": 7.084073078225803e-07,
+      "loss": 0.0963,
+      "reward": 0.972222238779068,
+      "reward_std": 0.6995532214641571,
+      "rewards/accuracy_reward": 0.19444444868713617,
+      "rewards/format_reward": 0.5833333432674408,
+      "step": 261
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2852.40283203125,
+      "epoch": 0.4490145672664953,
+      "grad_norm": 1.2071142196655273,
+      "kl": 0.5927734375,
+      "learning_rate": 7.05879227535395e-07,
+      "loss": 0.0551,
+      "reward": 1.208333358168602,
+      "reward_std": 0.6593898758292198,
+      "rewards/accuracy_reward": 0.33333333767950535,
+      "rewards/format_reward": 0.5416666641831398,
+      "step": 262
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2939.77783203125,
+      "epoch": 0.45072836332476435,
+      "grad_norm": 1.2289506196975708,
+      "kl": 0.587890625,
+      "learning_rate": 7.033455442031449e-07,
+      "loss": 0.174,
+      "reward": 0.8055555671453476,
+      "reward_std": 0.6164307221770287,
+      "rewards/accuracy_reward": 0.15277778171002865,
+      "rewards/format_reward": 0.5,
+      "step": 263
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2960.6806030273438,
+      "epoch": 0.4524421593830334,
+      "grad_norm": 0.7730751037597656,
+      "kl": 0.68212890625,
+      "learning_rate": 7.008063488985282e-07,
+      "loss": 0.0753,
+      "reward": 0.7083333283662796,
+      "reward_std": 0.5493002012372017,
+      "rewards/accuracy_reward": 0.09722222480922937,
+      "rewards/format_reward": 0.5138888955116272,
+      "step": 264
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3236.7500610351562,
+      "epoch": 0.4541559554413025,
+      "grad_norm": 0.514389157295227,
+      "kl": 0.73046875,
+      "learning_rate": 6.9826173289237e-07,
+      "loss": 0.1,
+      "reward": 1.111111119389534,
+      "reward_std": 0.9851955845952034,
+      "rewards/accuracy_reward": 0.31944444216787815,
+      "rewards/format_reward": 0.4722222313284874,
+      "step": 265
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3163.75,
+      "epoch": 0.45586975149957154,
+      "grad_norm": 0.6007187366485596,
+      "kl": 0.7626953125,
+      "learning_rate": 6.957117876503413e-07,
+      "loss": 0.1295,
+      "reward": 0.694444440305233,
+      "reward_std": 0.7118404507637024,
+      "rewards/accuracy_reward": 0.16666666977107525,
+      "rewards/format_reward": 0.3611111231148243,
+      "step": 266
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3049.6111450195312,
+      "epoch": 0.45758354755784064,
+      "grad_norm": 0.9735589027404785,
+      "kl": 0.712890625,
+      "learning_rate": 6.931566048296717e-07,
+      "loss": 0.0762,
+      "reward": 0.8750000149011612,
+      "reward_std": 0.6382264569401741,
+      "rewards/accuracy_reward": 0.15277778077870607,
+      "rewards/format_reward": 0.5694444552063942,
+      "step": 267
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2990.4862060546875,
+      "epoch": 0.4592973436161097,
+      "grad_norm": 1.0147572755813599,
+      "kl": 0.5908203125,
+      "learning_rate": 6.90596276275854e-07,
+      "loss": 0.1205,
+      "reward": 0.569444440305233,
+      "reward_std": 0.4945811964571476,
+      "rewards/accuracy_reward": 0.0555555559694767,
+      "rewards/format_reward": 0.4583333358168602,
+      "step": 268
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3235.236083984375,
+      "epoch": 0.46101113967437873,
+      "grad_norm": 0.45656487345695496,
+      "kl": 0.6708984375,
+      "learning_rate": 6.880308940193435e-07,
+      "loss": 0.1147,
+      "reward": 0.8194444477558136,
+      "reward_std": 0.9287731572985649,
+      "rewards/accuracy_reward": 0.1666666716337204,
+      "rewards/format_reward": 0.486111119389534,
+      "step": 269
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2900.7083740234375,
+      "epoch": 0.46272493573264784,
+      "grad_norm": 0.8187825083732605,
+      "kl": 0.5869140625,
+      "learning_rate": 6.854605502722496e-07,
+      "loss": 0.103,
+      "reward": 0.9861111044883728,
+      "reward_std": 0.6310887522995472,
+      "rewards/accuracy_reward": 0.25000000558793545,
+      "rewards/format_reward": 0.486111119389534,
+      "step": 270
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3006.5694580078125,
+      "epoch": 0.4644387317909169,
+      "grad_norm": 1.0085062980651855,
+      "kl": 0.65869140625,
+      "learning_rate": 6.828853374250211e-07,
+      "loss": 0.055,
+      "reward": 0.9166666716337204,
+      "reward_std": 0.6481297686696053,
+      "rewards/accuracy_reward": 0.19444444961845875,
+      "rewards/format_reward": 0.5277777835726738,
+      "step": 271
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2497.6944885253906,
+      "epoch": 0.4661525278491859,
+      "grad_norm": 0.7459636926651001,
+      "kl": 0.38525390625,
+      "learning_rate": 6.803053480431267e-07,
+      "loss": -0.0088,
+      "reward": 1.069444440305233,
+      "reward_std": 0.6437568441033363,
+      "rewards/accuracy_reward": 0.22222222574055195,
+      "rewards/format_reward": 0.6250000149011612,
+      "step": 272
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2684.9166870117188,
+      "epoch": 0.46786632390745503,
+      "grad_norm": 0.7339934706687927,
+      "kl": 0.428955078125,
+      "learning_rate": 6.777206748637253e-07,
+      "loss": 0.058,
+      "reward": 1.097222238779068,
+      "reward_std": 0.6862121671438217,
+      "rewards/accuracy_reward": 0.22222223225980997,
+      "rewards/format_reward": 0.6527777910232544,
+      "step": 273
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2835.638916015625,
+      "epoch": 0.4695801199657241,
+      "grad_norm": 0.6278461217880249,
+      "kl": 0.34814453125,
+      "learning_rate": 6.751314107923343e-07,
+      "loss": 0.0337,
+      "reward": 0.7638888880610466,
+      "reward_std": 0.6684231236577034,
+      "rewards/accuracy_reward": 0.06944444589316845,
+      "rewards/format_reward": 0.6250000074505806,
+      "step": 274
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3141.5833129882812,
+      "epoch": 0.4712939160239931,
+      "grad_norm": 0.5468356609344482,
+      "kl": 0.5888671875,
+      "learning_rate": 6.725376488994902e-07,
+      "loss": 0.0968,
+      "reward": 0.6527777761220932,
+      "reward_std": 0.5906506031751633,
+      "rewards/accuracy_reward": 0.1250000037252903,
+      "rewards/format_reward": 0.4027777910232544,
+      "step": 275
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2957.5139770507812,
+      "epoch": 0.4730077120822622,
+      "grad_norm": 0.6570262908935547,
+      "kl": 0.46240234375,
+      "learning_rate": 6.699394824174023e-07,
+      "loss": 0.051,
+      "reward": 0.8888888955116272,
+      "reward_std": 0.7196170538663864,
+      "rewards/accuracy_reward": 0.1666666679084301,
+      "rewards/format_reward": 0.555555559694767,
+      "step": 276
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3263.7777709960938,
+      "epoch": 0.47472150814053127,
+      "grad_norm": 0.5633208155632019,
+      "kl": 0.50390625,
+      "learning_rate": 6.673370047366016e-07,
+      "loss": 0.0475,
+      "reward": 0.7361111119389534,
+      "reward_std": 0.7856484949588776,
+      "rewards/accuracy_reward": 0.13888889271765947,
+      "rewards/format_reward": 0.4583333432674408,
+      "step": 277
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3001.0972900390625,
+      "epoch": 0.47643530419880037,
+      "grad_norm": 0.6549968719482422,
+      "kl": 0.4775390625,
+      "learning_rate": 6.647303094025848e-07,
+      "loss": 0.102,
+      "reward": 0.9027777910232544,
+      "reward_std": 0.6888429000973701,
+      "rewards/accuracy_reward": 0.16666667256504297,
+      "rewards/format_reward": 0.5694444477558136,
+      "step": 278
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3204.25,
+      "epoch": 0.4781491002570694,
+      "grad_norm": 0.4705544114112854,
+      "kl": 0.4609375,
+      "learning_rate": 6.621194901124511e-07,
+      "loss": 0.053,
+      "reward": 0.652777798473835,
+      "reward_std": 0.755700945854187,
+      "rewards/accuracy_reward": 0.09722222480922937,
+      "rewards/format_reward": 0.4583333358168602,
+      "step": 279
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2859.5,
+      "epoch": 0.47986289631533846,
+      "grad_norm": 0.7691783905029297,
+      "kl": 0.38427734375,
+      "learning_rate": 6.59504640711534e-07,
+      "loss": 0.1071,
+      "reward": 1.222222238779068,
+      "reward_std": 0.7407273650169373,
+      "rewards/accuracy_reward": 0.3055555606260896,
+      "rewards/format_reward": 0.611111119389534,
+      "step": 280
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2865.8194274902344,
+      "epoch": 0.48157669237360756,
+      "grad_norm": 0.3405319154262543,
+      "kl": 0.420166015625,
+      "learning_rate": 6.568858551900289e-07,
+      "loss": 0.0734,
+      "reward": 1.1527777910232544,
+      "reward_std": 0.8395373225212097,
+      "rewards/accuracy_reward": 0.2777777807787061,
+      "rewards/format_reward": 0.5972222313284874,
+      "step": 281
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2852.416717529297,
+      "epoch": 0.4832904884318766,
+      "grad_norm": 0.6291264295578003,
+      "kl": 0.361572265625,
+      "learning_rate": 6.542632276796142e-07,
+      "loss": 0.0659,
+      "reward": 1.0416666716337204,
+      "reward_std": 0.6899608969688416,
+      "rewards/accuracy_reward": 0.20833333488553762,
+      "rewards/format_reward": 0.625,
+      "step": 282
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3200.7777709960938,
+      "epoch": 0.48500428449014565,
+      "grad_norm": 0.5216673016548157,
+      "kl": 0.474609375,
+      "learning_rate": 6.516368524500672e-07,
+      "loss": 0.0597,
+      "reward": 0.819444477558136,
+      "reward_std": 0.6402985602617264,
+      "rewards/accuracy_reward": 0.16666666977107525,
+      "rewards/format_reward": 0.4861111268401146,
+      "step": 283
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3028.486083984375,
+      "epoch": 0.48671808054841476,
+      "grad_norm": 0.5700398087501526,
+      "kl": 0.423828125,
+      "learning_rate": 6.49006823905877e-07,
+      "loss": 0.0731,
+      "reward": 0.9722222313284874,
+      "reward_std": 0.6919921040534973,
+      "rewards/accuracy_reward": 0.1944444477558136,
+      "rewards/format_reward": 0.5833333507180214,
+      "step": 284
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3287.8333740234375,
+      "epoch": 0.4884318766066838,
+      "grad_norm": 0.46598854660987854,
+      "kl": 0.4638671875,
+      "learning_rate": 6.463732365828497e-07,
+      "loss": 0.0631,
+      "reward": 0.8194444440305233,
+      "reward_std": 0.7012417390942574,
+      "rewards/accuracy_reward": 0.1666666716337204,
+      "rewards/format_reward": 0.4861111156642437,
+      "step": 285
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3113.0694580078125,
+      "epoch": 0.49014567266495285,
+      "grad_norm": 1.0307719707489014,
+      "kl": 0.5625,
+      "learning_rate": 6.437361851447111e-07,
+      "loss": 0.067,
+      "reward": 0.8750000223517418,
+      "reward_std": 0.730349563062191,
+      "rewards/accuracy_reward": 0.1666666679084301,
+      "rewards/format_reward": 0.541666679084301,
+      "step": 286
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3025.7083740234375,
+      "epoch": 0.49185946872322195,
+      "grad_norm": 0.6258834600448608,
+      "kl": 0.46533203125,
+      "learning_rate": 6.410957643797038e-07,
+      "loss": 0.0885,
+      "reward": 0.8888889029622078,
+      "reward_std": 0.7127974927425385,
+      "rewards/accuracy_reward": 0.13888889364898205,
+      "rewards/format_reward": 0.6111111119389534,
+      "step": 287
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3184.236083984375,
+      "epoch": 0.493573264781491,
+      "grad_norm": 0.7940736413002014,
+      "kl": 0.62451171875,
+      "learning_rate": 6.384520691971805e-07,
+      "loss": 0.0912,
+      "reward": 0.541666679084301,
+      "reward_std": 0.5503551661968231,
+      "rewards/accuracy_reward": 0.055555556900799274,
+      "rewards/format_reward": 0.4305555745959282,
+      "step": 288
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2996.0833435058594,
+      "epoch": 0.4952870608397601,
+      "grad_norm": 0.49851223826408386,
+      "kl": 0.46435546875,
+      "learning_rate": 6.358051946241914e-07,
+      "loss": 0.0878,
+      "reward": 1.069444466382265,
+      "reward_std": 0.6114578023552895,
+      "rewards/accuracy_reward": 0.25000000186264515,
+      "rewards/format_reward": 0.5694444589316845,
+      "step": 289
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2852.77783203125,
+      "epoch": 0.49700085689802914,
+      "grad_norm": 0.7826417684555054,
+      "kl": 0.4990234375,
+      "learning_rate": 6.331552358020698e-07,
+      "loss": 0.1066,
+      "reward": 1.0277777761220932,
+      "reward_std": 0.7379468381404877,
+      "rewards/accuracy_reward": 0.22222222946584225,
+      "rewards/format_reward": 0.5833333432674408,
+      "step": 290
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3192.125,
+      "epoch": 0.4987146529562982,
+      "grad_norm": 1.0998865365982056,
+      "kl": 0.43701171875,
+      "learning_rate": 6.305022879830115e-07,
+      "loss": 0.0292,
+      "reward": 1.013888880610466,
+      "reward_std": 0.7805322334170341,
+      "rewards/accuracy_reward": 0.180555559694767,
+      "rewards/format_reward": 0.6527777910232544,
+      "step": 291
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2944.2083129882812,
+      "epoch": 0.5004284490145673,
+      "grad_norm": 0.6076076626777649,
+      "kl": 0.400634765625,
+      "learning_rate": 6.278464465266511e-07,
+      "loss": 0.0494,
+      "reward": 1.222222238779068,
+      "reward_std": 0.730344396084547,
+      "rewards/accuracy_reward": 0.29166666977107525,
+      "rewards/format_reward": 0.6388888955116272,
+      "step": 292
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2930.5,
+      "epoch": 0.5021422450728363,
+      "grad_norm": 0.40151283144950867,
+      "kl": 0.36669921875,
+      "learning_rate": 6.251878068966345e-07,
+      "loss": 0.0554,
+      "reward": 0.9444444645196199,
+      "reward_std": 0.6910195127129555,
+      "rewards/accuracy_reward": 0.16666667349636555,
+      "rewards/format_reward": 0.6111111212521791,
+      "step": 293
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2904.4166870117188,
+      "epoch": 0.5038560411311054,
+      "grad_norm": 0.5597663521766663,
+      "kl": 0.34423828125,
+      "learning_rate": 6.225264646571872e-07,
+      "loss": 0.0537,
+      "reward": 1.0555555820465088,
+      "reward_std": 0.6472389549016953,
+      "rewards/accuracy_reward": 0.19444444868713617,
+      "rewards/format_reward": 0.6666666641831398,
+      "step": 294
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2847.15283203125,
+      "epoch": 0.5055698371893744,
+      "grad_norm": 0.8492727279663086,
+      "kl": 0.300537109375,
+      "learning_rate": 6.198625154696796e-07,
+      "loss": 0.0718,
+      "reward": 1.2361111342906952,
+      "reward_std": 0.5978466831147671,
+      "rewards/accuracy_reward": 0.31944445613771677,
+      "rewards/format_reward": 0.5972222238779068,
+      "step": 295
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3084.6388549804688,
+      "epoch": 0.5072836332476436,
+      "grad_norm": 0.5291158556938171,
+      "kl": 0.41064453125,
+      "learning_rate": 6.171960550891878e-07,
+      "loss": 0.0733,
+      "reward": 1.222222238779068,
+      "reward_std": 0.7484849840402603,
+      "rewards/accuracy_reward": 0.3333333432674408,
+      "rewards/format_reward": 0.555555559694767,
+      "step": 296
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2997.5277709960938,
+      "epoch": 0.5089974293059126,
+      "grad_norm": 0.37727290391921997,
+      "kl": 0.372802734375,
+      "learning_rate": 6.145271793610529e-07,
+      "loss": 0.0517,
+      "reward": 1.0555555522441864,
+      "reward_std": 0.5442864708602428,
+      "rewards/accuracy_reward": 0.1944444477558136,
+      "rewards/format_reward": 0.6666666716337204,
+      "step": 297
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2720.1666870117188,
+      "epoch": 0.5107112253641817,
+      "grad_norm": 1.110034704208374,
+      "kl": 0.34375,
+      "learning_rate": 6.118559842174344e-07,
+      "loss": 0.0914,
+      "reward": 1.2777777761220932,
+      "reward_std": 0.8702788352966309,
+      "rewards/accuracy_reward": 0.2916666669771075,
+      "rewards/format_reward": 0.6944444477558136,
+      "step": 298
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3106.1389770507812,
+      "epoch": 0.5124250214224507,
+      "grad_norm": 0.4852089583873749,
+      "kl": 0.482421875,
+      "learning_rate": 6.091825656738635e-07,
+      "loss": 0.0899,
+      "reward": 1.0416666716337204,
+      "reward_std": 0.9716915190219879,
+      "rewards/accuracy_reward": 0.26388889364898205,
+      "rewards/format_reward": 0.5138888955116272,
+      "step": 299
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3082.6944580078125,
+      "epoch": 0.5141388174807198,
+      "grad_norm": 1.2142449617385864,
+      "kl": 0.52734375,
+      "learning_rate": 6.065070198257903e-07,
+      "loss": 0.111,
+      "reward": 1.0416666716337204,
+      "reward_std": 0.8163295686244965,
+      "rewards/accuracy_reward": 0.22222222294658422,
+      "rewards/format_reward": 0.5972222164273262,
+      "step": 300
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 583,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 6,
+  "trial_name": null,
+  "trial_params": null
+}