{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9986789960369881,
  "eval_steps": 500,
  "global_step": 378,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 1179.875,
      "epoch": 0.002642007926023778,
      "grad_norm": 0.4997229278087616,
      "kl": 0.0,
      "learning_rate": 2.6315789473684208e-08,
      "loss": 0.2467,
      "reward": 1.19921875,
      "reward_std": 0.13141997903585434,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.44921875,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1057.625,
      "epoch": 0.005284015852047556,
      "grad_norm": 0.5586327910423279,
      "kl": 0.0,
      "learning_rate": 5.2631578947368416e-08,
      "loss": 0.3641,
      "reward": 0.95703125,
      "reward_std": 0.12062124721705914,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.45703125,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1275.578125,
      "epoch": 0.007926023778071334,
      "grad_norm": 0.5091319680213928,
      "kl": 0.0001016855239868164,
      "learning_rate": 7.894736842105262e-08,
      "loss": 0.3625,
      "reward": 0.4140625,
      "reward_std": 0.13219169899821281,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 891.71875,
      "epoch": 0.010568031704095112,
      "grad_norm": 0.5754386782646179,
      "kl": 0.00015115737915039062,
      "learning_rate": 1.0526315789473683e-07,
      "loss": 0.3083,
      "reward": 0.99609375,
      "reward_std": 0.116029754281044,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.49609375,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1166.125,
      "epoch": 0.013210039630118891,
      "grad_norm": 0.5114976763725281,
      "kl": 0.00011730194091796875,
      "learning_rate": 1.3157894736842104e-07,
      "loss": 0.33,
      "reward": 0.9296875,
      "reward_std": 0.11507641524076462,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1035.703125,
      "epoch": 0.015852047556142668,
      "grad_norm": 0.7084254026412964,
      "kl": 0.00015091896057128906,
      "learning_rate": 1.5789473684210525e-07,
      "loss": 0.3363,
      "reward": 0.7265625,
      "reward_std": 0.12440211698412895,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 977.3125,
      "epoch": 0.018494055482166448,
      "grad_norm": 0.3633577525615692,
      "kl": 9.310245513916016e-05,
      "learning_rate": 1.8421052631578946e-07,
      "loss": 0.2085,
      "reward": 1.0,
      "reward_std": 0.13400040566921234,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 889.03125,
      "epoch": 0.021136063408190225,
      "grad_norm": 0.6363146901130676,
      "kl": 0.0001055002212524414,
      "learning_rate": 2.1052631578947366e-07,
      "loss": 0.3436,
      "reward": 0.984375,
      "reward_std": 0.11146603152155876,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.484375,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1114.28125,
      "epoch": 0.023778071334214,
      "grad_norm": 0.6163086295127869,
      "kl": 0.00010448694229125977,
      "learning_rate": 2.3684210526315787e-07,
      "loss": 0.387,
      "reward": 0.45703125,
      "reward_std": 0.11941792443394661,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.45703125,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1325.578125,
      "epoch": 0.026420079260237782,
      "grad_norm": 0.45183688402175903,
      "kl": 0.00015163421630859375,
      "learning_rate": 2.631578947368421e-07,
      "loss": 0.304,
      "reward": 0.91015625,
      "reward_std": 0.12797221168875694,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.41015625,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1116.671875,
      "epoch": 0.02906208718626156,
      "grad_norm": 0.5506221055984497,
      "kl": 0.0001614093780517578,
      "learning_rate": 2.894736842105263e-07,
      "loss": 0.2958,
      "reward": 0.7109375,
      "reward_std": 0.1341523937880993,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1107.546875,
      "epoch": 0.031704095112285335,
      "grad_norm": 0.423910528421402,
      "kl": 0.000125885009765625,
      "learning_rate": 3.157894736842105e-07,
      "loss": 0.2614,
      "reward": 0.9609375,
      "reward_std": 0.11495335027575493,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1075.453125,
      "epoch": 0.034346103038309116,
      "grad_norm": 0.6421769857406616,
      "kl": 0.0001609325408935547,
      "learning_rate": 3.4210526315789473e-07,
      "loss": 0.3804,
      "reward": 0.70703125,
      "reward_std": 0.11874673143029213,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.45703125,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1325.046875,
      "epoch": 0.036988110964332896,
      "grad_norm": 0.5751165151596069,
      "kl": 0.00011897087097167969,
      "learning_rate": 3.684210526315789e-07,
      "loss": 0.3482,
      "reward": 0.9296875,
      "reward_std": 0.15341992676258087,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1087.34375,
      "epoch": 0.03963011889035667,
      "grad_norm": 0.6110666394233704,
      "kl": 0.00010585784912109375,
      "learning_rate": 3.9473684210526315e-07,
      "loss": 0.3665,
      "reward": 0.95703125,
      "reward_std": 0.1287429742515087,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.45703125,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1314.15625,
      "epoch": 0.04227212681638045,
      "grad_norm": 0.5642758011817932,
      "kl": 0.00013065338134765625,
      "learning_rate": 4.2105263157894733e-07,
      "loss": 0.4046,
      "reward": 0.90625,
      "reward_std": 0.13578036427497864,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1289.359375,
      "epoch": 0.04491413474240423,
      "grad_norm": 0.4779168963432312,
      "kl": 8.845329284667969e-05,
      "learning_rate": 4.4736842105263156e-07,
      "loss": 0.2965,
      "reward": 1.421875,
      "reward_std": 0.12279411032795906,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1349.9375,
      "epoch": 0.047556142668428,
      "grad_norm": 0.4716605842113495,
      "kl": 0.00012004375457763672,
      "learning_rate": 4.7368421052631574e-07,
      "loss": 0.3496,
      "reward": 0.66796875,
      "reward_std": 0.14581536501646042,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.41796875,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 775.328125,
      "epoch": 0.05019815059445178,
      "grad_norm": 0.5275957584381104,
      "kl": 9.936094284057617e-05,
      "learning_rate": 5e-07,
      "loss": 0.3465,
      "reward": 0.734375,
      "reward_std": 0.08240052312612534,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.484375,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1096.671875,
      "epoch": 0.052840158520475564,
      "grad_norm": 0.622590959072113,
      "kl": 0.00011599063873291016,
      "learning_rate": 5.263157894736842e-07,
      "loss": 0.3991,
      "reward": 0.95703125,
      "reward_std": 0.09287451207637787,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.45703125,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1145.40625,
      "epoch": 0.05548216644649934,
      "grad_norm": 0.5628076195716858,
      "kl": 7.984042167663574e-05,
      "learning_rate": 5.526315789473684e-07,
      "loss": 0.3009,
      "reward": 0.7109375,
      "reward_std": 0.111817117780447,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 829.109375,
      "epoch": 0.05812417437252312,
      "grad_norm": 0.5253135561943054,
      "kl": 0.0001367330551147461,
      "learning_rate": 5.789473684210526e-07,
      "loss": 0.243,
      "reward": 1.4921875,
      "reward_std": 0.1498083807528019,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1060.53125,
      "epoch": 0.0607661822985469,
      "grad_norm": 0.624118983745575,
      "kl": 7.021427154541016e-05,
      "learning_rate": 6.052631578947368e-07,
      "loss": 0.4002,
      "reward": 1.21484375,
      "reward_std": 0.1456764042377472,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46484375,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 976.578125,
      "epoch": 0.06340819022457067,
      "grad_norm": 0.46764305233955383,
      "kl": 0.0001266002655029297,
      "learning_rate": 6.31578947368421e-07,
      "loss": 0.2928,
      "reward": 1.20703125,
      "reward_std": 0.096083864569664,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.45703125,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1207.015625,
      "epoch": 0.06605019815059446,
      "grad_norm": 0.39954128861427307,
      "kl": 0.00010007619857788086,
      "learning_rate": 6.578947368421053e-07,
      "loss": 0.1622,
      "reward": 0.953125,
      "reward_std": 0.15208648890256882,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 783.921875,
      "epoch": 0.06869220607661823,
      "grad_norm": 0.4758118689060211,
      "kl": 8.118152618408203e-05,
      "learning_rate": 6.842105263157895e-07,
      "loss": 0.2011,
      "reward": 0.96875,
      "reward_std": 0.07889671996235847,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 920.515625,
      "epoch": 0.071334214002642,
      "grad_norm": 0.7195703387260437,
      "kl": 9.21487808227539e-05,
      "learning_rate": 7.105263157894736e-07,
      "loss": 0.2896,
      "reward": 0.984375,
      "reward_std": 0.10958803817629814,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.484375,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1289.40625,
      "epoch": 0.07397622192866579,
      "grad_norm": 0.4253327548503876,
      "kl": 9.363889694213867e-05,
      "learning_rate": 7.368421052631578e-07,
      "loss": 0.0989,
      "reward": 0.9375,
      "reward_std": 0.1678653284907341,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1002.25,
      "epoch": 0.07661822985468957,
      "grad_norm": 0.7329438924789429,
      "kl": 0.0001462697982788086,
      "learning_rate": 7.631578947368421e-07,
      "loss": 0.4594,
      "reward": 0.95703125,
      "reward_std": 0.11983717978000641,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.45703125,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1205.53125,
      "epoch": 0.07926023778071334,
      "grad_norm": 0.7603439092636108,
      "kl": 0.00011014938354492188,
      "learning_rate": 7.894736842105263e-07,
      "loss": 0.4604,
      "reward": 0.9375,
      "reward_std": 0.1396191380918026,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1225.0625,
      "epoch": 0.08190224570673713,
      "grad_norm": 0.586107075214386,
      "kl": 0.0001385211944580078,
      "learning_rate": 8.157894736842105e-07,
      "loss": 0.2906,
      "reward": 0.7109375,
      "reward_std": 0.15029004588723183,
      "rewards/accuracy_reward": 0.265625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1195.609375,
      "epoch": 0.0845442536327609,
      "grad_norm": 0.5367782711982727,
      "kl": 0.00018310546875,
      "learning_rate": 8.421052631578947e-07,
      "loss": 0.3054,
      "reward": 1.18359375,
      "reward_std": 0.1250832974910736,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.43359375,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1193.53125,
      "epoch": 0.08718626155878467,
      "grad_norm": 0.6531537771224976,
      "kl": 0.0001990795135498047,
      "learning_rate": 8.684210526315789e-07,
      "loss": 0.382,
      "reward": 0.93359375,
      "reward_std": 0.10596734657883644,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.43359375,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 975.796875,
      "epoch": 0.08982826948480846,
      "grad_norm": 0.7079041004180908,
      "kl": 0.0002675056457519531,
      "learning_rate": 8.947368421052631e-07,
      "loss": 0.3162,
      "reward": 0.9921875,
      "reward_std": 0.11211910098791122,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1131.34375,
      "epoch": 0.09247027741083223,
      "grad_norm": 0.5116021037101746,
      "kl": 0.0003204345703125,
      "learning_rate": 9.210526315789473e-07,
      "loss": 0.3366,
      "reward": 1.19140625,
      "reward_std": 0.14293401315808296,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.44140625,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1189.828125,
      "epoch": 0.095112285336856,
      "grad_norm": 0.5107906460762024,
      "kl": 0.0003094673156738281,
      "learning_rate": 9.473684210526315e-07,
      "loss": 0.328,
      "reward": 1.20703125,
      "reward_std": 0.15370117127895355,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.45703125,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1482.453125,
      "epoch": 0.0977542932628798,
      "grad_norm": 0.46826329827308655,
      "kl": 0.0004634857177734375,
      "learning_rate": 9.736842105263158e-07,
      "loss": 0.2712,
      "reward": 1.16015625,
      "reward_std": 0.1653159111738205,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.41015625,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1108.578125,
      "epoch": 0.10039630118890357,
      "grad_norm": 0.5141110420227051,
      "kl": 0.0006732940673828125,
      "learning_rate": 1e-06,
      "loss": 0.1843,
      "reward": 0.97265625,
      "reward_std": 0.11588806286454201,
      "rewards/accuracy_reward": 0.515625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.45703125,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1196.65625,
      "epoch": 0.10303830911492734,
      "grad_norm": 0.5530170202255249,
      "kl": 0.000946044921875,
      "learning_rate": 9.999807902665155e-07,
      "loss": 0.2593,
      "reward": 0.9609375,
      "reward_std": 0.1273726001381874,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 822.328125,
      "epoch": 0.10568031704095113,
      "grad_norm": 0.6078239679336548,
      "kl": 0.001224517822265625,
      "learning_rate": 9.999231627061236e-07,
      "loss": 0.2837,
      "reward": 0.9921875,
      "reward_std": 0.10058118030428886,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 995.421875,
      "epoch": 0.1083223249669749,
      "grad_norm": 0.6204021573066711,
      "kl": 0.001720428466796875,
      "learning_rate": 9.998271222388693e-07,
      "loss": 0.4368,
      "reward": 1.2265625,
      "reward_std": 0.13393215090036392,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1254.515625,
      "epoch": 0.11096433289299867,
      "grad_norm": 0.6290051937103271,
      "kl": 0.0020294189453125,
      "learning_rate": 9.996926770643603e-07,
      "loss": 0.3358,
      "reward": 0.94921875,
      "reward_std": 0.13193362578749657,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.44921875,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 885.078125,
      "epoch": 0.11360634081902246,
      "grad_norm": 0.38145869970321655,
      "kl": 0.0020084381103515625,
      "learning_rate": 9.995198386610676e-07,
      "loss": 0.1421,
      "reward": 1.2421875,
      "reward_std": 0.09872931987047195,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1162.296875,
      "epoch": 0.11624834874504623,
      "grad_norm": 0.5801534056663513,
      "kl": 0.00255584716796875,
      "learning_rate": 9.993086217853452e-07,
      "loss": 0.3938,
      "reward": 0.9375,
      "reward_std": 0.12491972371935844,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 946.984375,
      "epoch": 0.11889035667107001,
      "grad_norm": 0.7080899477005005,
      "kl": 0.00287628173828125,
      "learning_rate": 9.990590444701706e-07,
      "loss": 0.3176,
      "reward": 0.71484375,
      "reward_std": 0.07072163559496403,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46484375,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1258.78125,
      "epoch": 0.1215323645970938,
      "grad_norm": 0.6584539413452148,
      "kl": 0.00337982177734375,
      "learning_rate": 9.987711280236046e-07,
      "loss": 0.3364,
      "reward": 0.9296875,
      "reward_std": 0.10684756934642792,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1006.984375,
      "epoch": 0.12417437252311757,
      "grad_norm": 0.5412375926971436,
      "kl": 0.003643035888671875,
      "learning_rate": 9.984448970269725e-07,
      "loss": 0.2438,
      "reward": 1.25390625,
      "reward_std": 0.16918476670980453,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.50390625,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1165.140625,
      "epoch": 0.12681638044914134,
      "grad_norm": 0.5502119064331055,
      "kl": 0.00435638427734375,
      "learning_rate": 9.980803793327655e-07,
      "loss": 0.329,
      "reward": 0.73046875,
      "reward_std": 0.17235729470849037,
      "rewards/accuracy_reward": 0.265625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46484375,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1094.59375,
      "epoch": 0.12945838837516513,
      "grad_norm": 0.6746593713760376,
      "kl": 0.0046234130859375,
      "learning_rate": 9.976776060622625e-07,
      "loss": 0.2585,
      "reward": 0.68359375,
      "reward_std": 0.11046826094388962,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.43359375,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 894.875,
      "epoch": 0.13210039630118892,
      "grad_norm": 0.6030331254005432,
      "kl": 0.0045623779296875,
      "learning_rate": 9.972366116028733e-07,
      "loss": 0.1373,
      "reward": 1.2265625,
      "reward_std": 0.11612267419695854,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 660.078125,
      "epoch": 0.13474240422721268,
      "grad_norm": 0.7342778444290161,
      "kl": 0.00536346435546875,
      "learning_rate": 9.96757433605202e-07,
      "loss": 0.2687,
      "reward": 1.26171875,
      "reward_std": 0.11859130859375,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51171875,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1063.71875,
      "epoch": 0.13738441215323646,
      "grad_norm": 0.7268034219741821,
      "kl": 0.00653076171875,
      "learning_rate": 9.962401129798343e-07,
      "loss": 0.3436,
      "reward": 0.98046875,
      "reward_std": 0.15140536800026894,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.48046875,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1265.96875,
      "epoch": 0.14002642007926025,
      "grad_norm": 0.7652710676193237,
      "kl": 0.00766754150390625,
      "learning_rate": 9.956846938938422e-07,
      "loss": 0.4375,
      "reward": 0.91015625,
      "reward_std": 0.1307620257139206,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.41015625,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1342.03125,
      "epoch": 0.142668428005284,
      "grad_norm": 0.6607176065444946,
      "kl": 0.0090179443359375,
      "learning_rate": 9.950912237670157e-07,
      "loss": 0.3436,
      "reward": 0.90234375,
      "reward_std": 0.1162625178694725,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40234375,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1300.421875,
      "epoch": 0.1453104359313078,
      "grad_norm": 0.6878055930137634,
      "kl": 0.01092529296875,
      "learning_rate": 9.944597532678119e-07,
      "loss": 0.3859,
      "reward": 1.1640625,
      "reward_std": 0.1533336602151394,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1351.71875,
      "epoch": 0.14795244385733158,
      "grad_norm": 0.6881883144378662,
      "kl": 0.01397705078125,
      "learning_rate": 9.93790336309031e-07,
      "loss": 0.3671,
      "reward": 0.92578125,
      "reward_std": 0.15761961415410042,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.42578125,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 710.25,
      "epoch": 0.15059445178335534,
      "grad_norm": 0.5193164348602295,
      "kl": 0.0154571533203125,
      "learning_rate": 9.930830300432126e-07,
      "loss": 0.1832,
      "reward": 1.01953125,
      "reward_std": 0.11765347048640251,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51953125,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1319.109375,
      "epoch": 0.15323645970937913,
      "grad_norm": 0.6145569086074829,
      "kl": 0.0148468017578125,
      "learning_rate": 9.923378948577558e-07,
      "loss": 0.3036,
      "reward": 0.9375,
      "reward_std": 0.1474018730223179,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1229.484375,
      "epoch": 0.15587846763540292,
      "grad_norm": 0.6062135100364685,
      "kl": 0.0187530517578125,
      "learning_rate": 9.915549943697644e-07,
      "loss": 0.3039,
      "reward": 0.92578125,
      "reward_std": 0.12412451207637787,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.42578125,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1122.96875,
      "epoch": 0.15852047556142668,
      "grad_norm": 0.7750731110572815,
      "kl": 0.019989013671875,
      "learning_rate": 9.907343954206146e-07,
      "loss": 0.4269,
      "reward": 0.4609375,
      "reward_std": 0.15149712190032005,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1289.796875,
      "epoch": 0.16116248348745046,
      "grad_norm": 0.4260408282279968,
      "kl": 0.023284912109375,
      "learning_rate": 9.898761680702495e-07,
      "loss": 0.2105,
      "reward": 0.66015625,
      "reward_std": 0.10409127548336983,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.41015625,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1395.359375,
      "epoch": 0.16380449141347425,
      "grad_norm": 0.4302825629711151,
      "kl": 0.022216796875,
      "learning_rate": 9.889803855911965e-07,
      "loss": 0.2882,
      "reward": 0.69140625,
      "reward_std": 0.17329547554254532,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.44140625,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1214.203125,
      "epoch": 0.166446499339498,
      "grad_norm": 0.5709892511367798,
      "kl": 0.025421142578125,
      "learning_rate": 9.880471244623118e-07,
      "loss": 0.2752,
      "reward": 0.96484375,
      "reward_std": 0.16381771862506866,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46484375,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1456.640625,
      "epoch": 0.1690885072655218,
      "grad_norm": 0.4366983473300934,
      "kl": 0.03094482421875,
      "learning_rate": 9.87076464362251e-07,
      "loss": 0.1409,
      "reward": 1.21484375,
      "reward_std": 0.1545065976679325,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46484375,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 907.890625,
      "epoch": 0.17173051519154559,
      "grad_norm": 0.5789319276809692,
      "kl": 0.0296630859375,
      "learning_rate": 9.860684881626674e-07,
      "loss": 0.223,
      "reward": 1.0234375,
      "reward_std": 0.18188364803791046,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1124.03125,
      "epoch": 0.17437252311756934,
      "grad_norm": 0.8789018988609314,
      "kl": 0.03033447265625,
      "learning_rate": 9.850232819211343e-07,
      "loss": -0.0662,
      "reward": 0.9609375,
      "reward_std": 0.16317331418395042,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1024.5,
      "epoch": 0.17701453104359313,
      "grad_norm": 0.7724674344062805,
      "kl": 0.03656005859375,
      "learning_rate": 9.839409348738e-07,
      "loss": 0.2921,
      "reward": 1.21875,
      "reward_std": 0.12279859185218811,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1002.6875,
      "epoch": 0.17965653896961692,
      "grad_norm": 1.0319114923477173,
      "kl": 0.043212890625,
      "learning_rate": 9.828215394277686e-07,
      "loss": 0.3121,
      "reward": 0.97265625,
      "reward_std": 0.13220234587788582,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.47265625,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1255.0625,
      "epoch": 0.18229854689564068,
      "grad_norm": 0.7915776371955872,
      "kl": 0.041290283203125,
      "learning_rate": 9.816651911532093e-07,
      "loss": 0.3672,
      "reward": 0.93359375,
      "reward_std": 0.16574888676404953,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.43359375,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1128.546875,
      "epoch": 0.18494055482166447,
      "grad_norm": 0.577376127243042,
      "kl": 0.040679931640625,
      "learning_rate": 9.804719887751984e-07,
      "loss": 0.1898,
      "reward": 1.0078125,
      "reward_std": 0.17545727640390396,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5078125,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1146.234375,
      "epoch": 0.18758256274768825,
      "grad_norm": 0.5707401633262634,
      "kl": 0.034698486328125,
      "learning_rate": 9.792420341652901e-07,
      "loss": 0.269,
      "reward": 1.1796875,
      "reward_std": 0.11014671996235847,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1104.234375,
      "epoch": 0.190224570673712,
      "grad_norm": 0.5689163208007812,
      "kl": 0.0369873046875,
      "learning_rate": 9.779754323328192e-07,
      "loss": 0.3013,
      "reward": 0.73046875,
      "reward_std": 0.1631980687379837,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.48046875,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1476.96875,
      "epoch": 0.1928665785997358,
      "grad_norm": 0.5846036672592163,
      "kl": 0.033660888671875,
      "learning_rate": 9.766722914159345e-07,
      "loss": 0.2798,
      "reward": 0.8984375,
      "reward_std": 0.1427699662744999,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1340.578125,
      "epoch": 0.1955085865257596,
      "grad_norm": 0.4723777174949646,
      "kl": 0.035400390625,
      "learning_rate": 9.753327226723687e-07,
      "loss": 0.2281,
      "reward": 0.64453125,
      "reward_std": 0.09241959825158119,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.39453125,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1429.40625,
      "epoch": 0.19815059445178335,
      "grad_norm": 0.6316815614700317,
      "kl": 0.03790283203125,
      "learning_rate": 9.73956840469937e-07,
      "loss": 0.2594,
      "reward": 1.1640625,
      "reward_std": 0.14494511112570763,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1395.609375,
      "epoch": 0.20079260237780713,
      "grad_norm": 0.4536829888820648,
      "kl": 0.036865234375,
      "learning_rate": 9.725447622767754e-07,
      "loss": 0.257,
      "reward": 1.24609375,
      "reward_std": 0.24476346373558044,
      "rewards/accuracy_reward": 0.765625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.48046875,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1369.703125,
      "epoch": 0.20343461030383092,
      "grad_norm": 0.519792914390564,
      "kl": 0.04010009765625,
      "learning_rate": 9.710966086513085e-07,
      "loss": 0.2693,
      "reward": 0.93359375,
      "reward_std": 0.15936565026640892,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.43359375,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1090.53125,
      "epoch": 0.20607661822985468,
      "grad_norm": 0.7418442368507385,
      "kl": 0.04974365234375,
      "learning_rate": 9.6961250323196e-07,
      "loss": 0.3581,
      "reward": 1.203125,
      "reward_std": 0.14408493414521217,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1075.609375,
      "epoch": 0.20871862615587847,
      "grad_norm": 0.4650673270225525,
      "kl": 0.046630859375,
      "learning_rate": 9.680925727265944e-07,
      "loss": 0.1385,
      "reward": 0.984375,
      "reward_std": 0.13037987425923347,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.484375,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1463.03125,
      "epoch": 0.21136063408190225,
      "grad_norm": 0.44249987602233887,
      "kl": 0.047119140625,
      "learning_rate": 9.665369469017002e-07,
      "loss": 0.1594,
      "reward": 0.8984375,
      "reward_std": 0.16113372519612312,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1128.3125,
      "epoch": 0.21400264200792601,
      "grad_norm": 0.543846070766449,
      "kl": 0.05157470703125,
      "learning_rate": 9.649457585713108e-07,
      "loss": 0.2237,
      "reward": 1.234375,
      "reward_std": 0.1662597917020321,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.484375,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 960.765625,
      "epoch": 0.2166446499339498,
      "grad_norm": 0.7787006497383118,
      "kl": 0.0552978515625,
      "learning_rate": 9.633191435856653e-07,
      "loss": 0.3572,
      "reward": 1.2109375,
      "reward_std": 0.12929406948387623,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1094.390625,
      "epoch": 0.2192866578599736,
      "grad_norm": 0.9358471632003784,
      "kl": 0.060302734375,
      "learning_rate": 9.616572408196093e-07,
      "loss": 0.3621,
      "reward": 0.73046875,
      "reward_std": 0.18469755724072456,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.48046875,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 868.265625,
      "epoch": 0.22192866578599735,
      "grad_norm": 1.0493205785751343,
      "kl": 0.06304931640625,
      "learning_rate": 9.599601921607397e-07,
      "loss": 0.3486,
      "reward": 0.5078125,
      "reward_std": 0.16107311472296715,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5078125,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1090.9375,
      "epoch": 0.22457067371202113,
      "grad_norm": 0.9199777245521545,
      "kl": 0.06231689453125,
      "learning_rate": 9.582281424972892e-07,
      "loss": 0.3608,
      "reward": 0.96484375,
      "reward_std": 0.129608154296875,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46484375,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1144.140625,
      "epoch": 0.22721268163804492,
      "grad_norm": 0.7876753807067871,
      "kl": 0.067138671875,
      "learning_rate": 9.56461239705758e-07,
      "loss": 0.2158,
      "reward": 0.44921875,
      "reward_std": 0.11367761343717575,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.44921875,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1346.328125,
      "epoch": 0.22985468956406868,
      "grad_norm": 0.8156364560127258,
      "kl": 0.06951904296875,
      "learning_rate": 9.546596346382864e-07,
      "loss": 0.2484,
      "reward": 0.92578125,
      "reward_std": 0.14216843992471695,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.42578125,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 906.40625,
      "epoch": 0.23249669749009247,
      "grad_norm": 0.6532436013221741,
      "kl": 0.083984375,
      "learning_rate": 9.528234811097781e-07,
      "loss": 0.1984,
      "reward": 1.24609375,
      "reward_std": 0.10012037679553032,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.49609375,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1103.65625,
      "epoch": 0.23513870541611626,
      "grad_norm": 0.6433841586112976,
      "kl": 0.0770263671875,
      "learning_rate": 9.509529358847654e-07,
      "loss": 0.1822,
      "reward": 0.70703125,
      "reward_std": 0.12630900368094444,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.45703125,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1273.546875,
      "epoch": 0.23778071334214002,
      "grad_norm": 1.185502529144287,
      "kl": 0.106201171875,
      "learning_rate": 9.490481586640278e-07,
      "loss": 0.3498,
      "reward": 0.91796875,
      "reward_std": 0.14778802916407585,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.41796875,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1223.328125,
      "epoch": 0.2404227212681638,
      "grad_norm": 0.6358450055122375,
      "kl": 0.1009521484375,
      "learning_rate": 9.47109312070955e-07,
      "loss": 0.1773,
      "reward": 0.74609375,
      "reward_std": 0.18448476120829582,
      "rewards/accuracy_reward": 0.265625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.48046875,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 700.5625,
      "epoch": 0.2430647291941876,
      "grad_norm": 0.750359058380127,
      "kl": 0.1322021484375,
      "learning_rate": 9.45136561637664e-07,
      "loss": 0.1891,
      "reward": 1.046875,
      "reward_std": 0.14496402069926262,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 863.90625,
      "epoch": 0.24570673712021135,
      "grad_norm": 0.557322084903717,
      "kl": 0.1099853515625,
      "learning_rate": 9.431300757908663e-07,
      "loss": 0.1089,
      "reward": 1.30078125,
      "reward_std": 0.15019455552101135,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55078125,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 970.015625,
      "epoch": 0.24834874504623514,
      "grad_norm": 0.731271505355835,
      "kl": 0.12158203125,
      "learning_rate": 9.410900258374876e-07,
      "loss": 0.1692,
      "reward": 0.76953125,
      "reward_std": 0.17832617834210396,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51953125,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 924.6875,
      "epoch": 0.2509907529722589,
      "grad_norm": 1.327541708946228,
      "kl": 0.14990234375,
      "learning_rate": 9.390165859500435e-07,
      "loss": 0.2367,
      "reward": 0.5234375,
      "reward_std": 0.1663740910589695,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1177.421875,
      "epoch": 0.2536327608982827,
      "grad_norm": 1.7957454919815063,
      "kl": 0.165771484375,
      "learning_rate": 9.369099331517676e-07,
      "loss": 0.3655,
      "reward": 0.9453125,
      "reward_std": 0.17608627676963806,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1124.71875,
      "epoch": 0.2562747688243065,
      "grad_norm": 1.353155493736267,
      "kl": 0.1519775390625,
      "learning_rate": 9.34770247301499e-07,
      "loss": 0.2683,
      "reward": 1.2109375,
      "reward_std": 0.11838950589299202,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 710.703125,
      "epoch": 0.25891677675033026,
      "grad_norm": 1.821932077407837,
      "kl": 0.19970703125,
      "learning_rate": 9.325977110783263e-07,
      "loss": 0.1213,
      "reward": 1.52734375,
      "reward_std": 0.14770140498876572,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.52734375,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 910.125,
      "epoch": 0.261558784676354,
      "grad_norm": 0.8406642079353333,
      "kl": 0.185546875,
      "learning_rate": 9.30392509965991e-07,
      "loss": 0.1623,
      "reward": 1.015625,
      "reward_std": 0.1544700786471367,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.515625,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1078.859375,
      "epoch": 0.26420079260237783,
      "grad_norm": 1.6371651887893677,
      "kl": 0.225341796875,
      "learning_rate": 9.281548322370517e-07,
      "loss": 0.2703,
      "reward": 0.72265625,
      "reward_std": 0.14984130859375,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.47265625,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 532.15625,
      "epoch": 0.2668428005284016,
      "grad_norm": 2.1254074573516846,
      "kl": 0.246826171875,
      "learning_rate": 9.258848689368094e-07,
      "loss": 0.2214,
      "reward": 1.2578125,
      "reward_std": 0.10374833643436432,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5078125,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 848.5625,
      "epoch": 0.26948480845442535,
      "grad_norm": 2.5907938480377197,
      "kl": 0.2958984375,
      "learning_rate": 9.235828138669978e-07,
      "loss": 0.3198,
      "reward": 1.01171875,
      "reward_std": 0.137377567589283,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51171875,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1073.3125,
      "epoch": 0.27212681638044917,
      "grad_norm": 2.286487102508545,
      "kl": 0.2607421875,
      "learning_rate": 9.21248863569236e-07,
      "loss": 0.3082,
      "reward": 0.97265625,
      "reward_std": 0.15867146104574203,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.47265625,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 858.796875,
      "epoch": 0.2747688243064729,
      "grad_norm": 1.7667056322097778,
      "kl": 0.33837890625,
      "learning_rate": 9.188832173082495e-07,
      "loss": 0.2436,
      "reward": 0.71875,
      "reward_std": 0.10251419246196747,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1045.796875,
      "epoch": 0.2774108322324967,
      "grad_norm": 2.42461895942688,
      "kl": 0.40380859375,
      "learning_rate": 9.164860770548567e-07,
      "loss": 0.2974,
      "reward": 0.9921875,
      "reward_std": 0.16395077854394913,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 893.453125,
      "epoch": 0.2800528401585205,
      "grad_norm": 10.34216594696045,
      "kl": 0.474609375,
      "learning_rate": 9.140576474687263e-07,
      "loss": 0.294,
      "reward": 0.97265625,
      "reward_std": 0.1429976001381874,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.47265625,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1402.546875,
      "epoch": 0.28269484808454426,
      "grad_norm": 5.165650367736816,
      "kl": 0.5849609375,
      "learning_rate": 9.11598135880903e-07,
      "loss": 0.3739,
      "reward": 0.6484375,
      "reward_std": 0.16659503430128098,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 959.703125,
      "epoch": 0.285336856010568,
      "grad_norm": 5.434719562530518,
      "kl": 0.6767578125,
      "learning_rate": 9.091077522761078e-07,
      "loss": 0.421,
      "reward": 0.9765625,
      "reward_std": 0.13730589486658573,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1104.828125,
      "epoch": 0.28797886393659183,
      "grad_norm": 1.7607016563415527,
      "kl": 0.40234375,
      "learning_rate": 9.065867092748082e-07,
      "loss": 0.205,
      "reward": 0.71875,
      "reward_std": 0.16618655994534492,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1261.84375,
      "epoch": 0.2906208718626156,
      "grad_norm": 3.3362314701080322,
      "kl": 0.57373046875,
      "learning_rate": 9.040352221150674e-07,
      "loss": 0.3039,
      "reward": 0.71875,
      "reward_std": 0.2016766332089901,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 850.640625,
      "epoch": 0.29326287978863935,
      "grad_norm": 3.9499456882476807,
      "kl": 0.54296875,
      "learning_rate": 9.014535086341669e-07,
      "loss": 0.3804,
      "reward": 1.234375,
      "reward_std": 0.14762691780924797,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.484375,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 871.765625,
      "epoch": 0.29590488771466317,
      "grad_norm": 4.223949432373047,
      "kl": 0.5234375,
      "learning_rate": 8.988417892500083e-07,
      "loss": 0.3621,
      "reward": 1.2734375,
      "reward_std": 0.18184370175004005,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 711.078125,
      "epoch": 0.2985468956406869,
      "grad_norm": 10.757521629333496,
      "kl": 0.53955078125,
      "learning_rate": 8.962002869422955e-07,
      "loss": 0.6943,
      "reward": 0.484375,
      "reward_std": 0.17551938444375992,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.484375,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 485.96875,
      "epoch": 0.3011889035667107,
      "grad_norm": 6.041623592376709,
      "kl": 0.59326171875,
      "learning_rate": 8.935292272334963e-07,
      "loss": 0.4734,
      "reward": 0.76953125,
      "reward_std": 0.13621540740132332,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51953125,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 615.203125,
      "epoch": 0.3038309114927345,
      "grad_norm": 2.360245943069458,
      "kl": 0.60302734375,
      "learning_rate": 8.908288381695892e-07,
      "loss": 0.2661,
      "reward": 1.2578125,
      "reward_std": 0.1489735022187233,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5078125,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 851.28125,
      "epoch": 0.30647291941875826,
      "grad_norm": 6.270340442657471,
      "kl": 0.8740234375,
      "learning_rate": 8.88099350300593e-07,
      "loss": 0.5072,
      "reward": 0.73046875,
      "reward_std": 0.15848717093467712,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.48046875,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1406.65625,
      "epoch": 0.309114927344782,
      "grad_norm": 4.970353126525879,
      "kl": 1.427734375,
      "learning_rate": 8.853409966608831e-07,
      "loss": 0.3739,
      "reward": 0.65234375,
      "reward_std": 0.15436260029673576,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40234375,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 682.796875,
      "epoch": 0.31175693527080584,
      "grad_norm": 11.649397850036621,
      "kl": 1.416015625,
      "learning_rate": 8.825540127492965e-07,
      "loss": 0.582,
      "reward": 1.2734375,
      "reward_std": 0.16201764903962612,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 346.234375,
      "epoch": 0.3143989431968296,
      "grad_norm": 6.038275241851807,
      "kl": 1.6640625,
      "learning_rate": 8.797386365090252e-07,
      "loss": 0.4335,
      "reward": 1.3046875,
      "reward_std": 0.16278167814016342,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5546875,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 878.03125,
      "epoch": 0.31704095112285335,
      "grad_norm": 12.164133071899414,
      "kl": 2.13671875,
      "learning_rate": 8.768951083073009e-07,
      "loss": 0.8115,
      "reward": 0.9921875,
      "reward_std": 0.1910713165998459,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.671875,
      "epoch": 0.31968295904887717,
      "grad_norm": 9.305420875549316,
      "kl": 2.453125,
      "learning_rate": 8.740236709148745e-07,
      "loss": 0.6232,
      "reward": 1.29296875,
      "reward_std": 0.1861564740538597,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.54296875,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.578125,
      "epoch": 0.32232496697490093,
      "grad_norm": 11.043706893920898,
      "kl": 2.4150390625,
      "learning_rate": 8.711245694852886e-07,
      "loss": 0.4605,
      "reward": 1.296875,
      "reward_std": 0.20820768922567368,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 501.625,
      "epoch": 0.3249669749009247,
      "grad_norm": 10.729813575744629,
      "kl": 2.490234375,
      "learning_rate": 8.681980515339463e-07,
      "loss": 0.6364,
      "reward": 0.8359375,
      "reward_std": 0.23206235468387604,
      "rewards/accuracy_reward": 0.265625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 466.9375,
      "epoch": 0.3276089828269485,
      "grad_norm": 7.306431770324707,
      "kl": 2.515625,
      "learning_rate": 8.652443669169809e-07,
      "loss": 0.5031,
      "reward": 0.5625,
      "reward_std": 0.18624207936227322,
      "rewards/accuracy_reward": 0.015625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 435.640625,
      "epoch": 0.33025099075297226,
      "grad_norm": 9.731188774108887,
      "kl": 3.28515625,
      "learning_rate": 8.622637678099224e-07,
      "loss": 0.7344,
      "reward": 1.01171875,
      "reward_std": 0.16986817121505737,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51171875,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 291.671875,
      "epoch": 0.332892998678996,
      "grad_norm": 11.137627601623535,
      "kl": 3.392578125,
      "learning_rate": 8.592565086861681e-07,
      "loss": 0.3762,
      "reward": 1.01953125,
      "reward_std": 0.1285141110420227,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51953125,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 543.8125,
      "epoch": 0.33553500660501984,
      "grad_norm": 16.820133209228516,
      "kl": 3.1875,
      "learning_rate": 8.562228462952576e-07,
      "loss": 0.2899,
      "reward": 1.28125,
      "reward_std": 0.1833672672510147,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53125,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 232.859375,
      "epoch": 0.3381770145310436,
      "grad_norm": 10.55738353729248,
      "kl": 2.62939453125,
      "learning_rate": 8.531630396409507e-07,
      "loss": 0.2709,
      "reward": 1.06640625,
      "reward_std": 0.12935607135295868,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 718.671875,
      "epoch": 0.34081902245706736,
      "grad_norm": 10.954379081726074,
      "kl": 3.91015625,
      "learning_rate": 8.500773499591156e-07,
      "loss": 0.3251,
      "reward": 0.5078125,
      "reward_std": 0.10781864821910858,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5078125,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 489.109375,
      "epoch": 0.34346103038309117,
      "grad_norm": 10.081979751586914,
      "kl": 2.50390625,
      "learning_rate": 8.469660406954252e-07,
      "loss": 0.4498,
      "reward": 0.796875,
      "reward_std": 0.20939984917640686,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 377.484375,
      "epoch": 0.34610303830911493,
      "grad_norm": 4.734899520874023,
      "kl": 1.208984375,
      "learning_rate": 8.438293774828649e-07,
      "loss": 0.2461,
      "reward": 1.3046875,
      "reward_std": 0.16797470301389694,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5546875,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 353.15625,
      "epoch": 0.3487450462351387,
      "grad_norm": 3.945875883102417,
      "kl": 1.7080078125,
      "learning_rate": 8.406676281190542e-07,
      "loss": 0.2267,
      "reward": 0.83984375,
      "reward_std": 0.172641359269619,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58984375,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 632.921875,
      "epoch": 0.3513870541611625,
      "grad_norm": 26.704730987548828,
      "kl": 1.767578125,
      "learning_rate": 8.374810625433825e-07,
      "loss": 0.7894,
      "reward": 1.02734375,
      "reward_std": 0.21192153729498386,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.52734375,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 478.828125,
      "epoch": 0.35402906208718626,
      "grad_norm": 23.016502380371094,
      "kl": 1.65234375,
      "learning_rate": 8.342699528139628e-07,
      "loss": 0.5162,
      "reward": 1.015625,
      "reward_std": 0.1322025004774332,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.515625,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 613.46875,
      "epoch": 0.35667107001321,
      "grad_norm": 5.931519985198975,
      "kl": 2.02734375,
      "learning_rate": 8.310345730844047e-07,
      "loss": 0.4553,
      "reward": 1.3125,
      "reward_std": 0.21167393401265144,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5625,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 500.46875,
      "epoch": 0.35931307793923384,
      "grad_norm": 7.461983680725098,
      "kl": 1.9765625,
      "learning_rate": 8.277751995804067e-07,
      "loss": 0.3654,
      "reward": 1.0234375,
      "reward_std": 0.1544732078909874,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 439.890625,
      "epoch": 0.3619550858652576,
      "grad_norm": 3.8175482749938965,
      "kl": 2.041015625,
      "learning_rate": 8.244921105761755e-07,
      "loss": 0.3475,
      "reward": 1.07421875,
      "reward_std": 0.23262840881943703,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.57421875,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 442.796875,
      "epoch": 0.36459709379128136,
      "grad_norm": 11.061271667480469,
      "kl": 1.546875,
      "learning_rate": 8.211855863706654e-07,
      "loss": 0.5592,
      "reward": 1.2890625,
      "reward_std": 0.17124063521623611,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5390625,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 334.875,
      "epoch": 0.36723910171730517,
      "grad_norm": 12.917343139648438,
      "kl": 2.42578125,
      "learning_rate": 8.178559092636484e-07,
      "loss": 0.1005,
      "reward": 0.6015625,
      "reward_std": 0.1888568513095379,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6015625,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 815.125,
      "epoch": 0.36988110964332893,
      "grad_norm": 4.946498394012451,
      "kl": 2.6484375,
      "learning_rate": 8.145033635316128e-07,
      "loss": 0.4205,
      "reward": 0.51171875,
      "reward_std": 0.19404659420251846,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51171875,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 358.640625,
      "epoch": 0.3725231175693527,
      "grad_norm": 4.1423869132995605,
      "kl": 2.376953125,
      "learning_rate": 8.111282354034921e-07,
      "loss": 0.362,
      "reward": 1.0546875,
      "reward_std": 0.1854284517467022,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5546875,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 220.828125,
      "epoch": 0.3751651254953765,
      "grad_norm": 14.8277006149292,
      "kl": 3.98828125,
      "learning_rate": 8.077308130362273e-07,
      "loss": 0.1853,
      "reward": 1.0390625,
      "reward_std": 0.12213464453816414,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5390625,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 880.515625,
      "epoch": 0.37780713342140027,
      "grad_norm": 4.2313103675842285,
      "kl": 2.3984375,
      "learning_rate": 8.043113864901663e-07,
      "loss": 0.4005,
      "reward": 1.20703125,
      "reward_std": 0.1507197804749012,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.45703125,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 561.265625,
      "epoch": 0.380449141347424,
      "grad_norm": 7.7739458084106445,
      "kl": 2.126953125,
      "learning_rate": 8.008702477042985e-07,
      "loss": 0.4939,
      "reward": 1.3203125,
      "reward_std": 0.20398560166358948,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 698.015625,
      "epoch": 0.38309114927344784,
      "grad_norm": 6.112682342529297,
      "kl": 2.115234375,
      "learning_rate": 7.974076904713301e-07,
      "loss": 0.4279,
      "reward": 0.73828125,
      "reward_std": 0.09649410098791122,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.48828125,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 359.640625,
      "epoch": 0.3857331571994716,
      "grad_norm": 11.499645233154297,
      "kl": 1.916015625,
      "learning_rate": 7.939240104126022e-07,
      "loss": 0.4661,
      "reward": 1.04296875,
      "reward_std": 0.1618601270020008,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.54296875,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 364.5,
      "epoch": 0.38837516512549536,
      "grad_norm": 5.250813961029053,
      "kl": 2.73046875,
      "learning_rate": 7.904195049528497e-07,
      "loss": 0.4228,
      "reward": 1.09765625,
      "reward_std": 0.2164350003004074,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59765625,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 624.75,
      "epoch": 0.3910171730515192,
      "grad_norm": 12.445371627807617,
      "kl": 1.9091796875,
      "learning_rate": 7.8689447329481e-07,
      "loss": 0.5554,
      "reward": 1.015625,
      "reward_std": 0.20019326359033585,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.515625,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 388.015625,
      "epoch": 0.39365918097754293,
      "grad_norm": 14.60313606262207,
      "kl": 3.12890625,
      "learning_rate": 7.833492163936773e-07,
      "loss": 0.2208,
      "reward": 1.01953125,
      "reward_std": 0.15205424278974533,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51953125,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 278.0625,
      "epoch": 0.3963011889035667,
      "grad_norm": 9.847626686096191,
      "kl": 2.1591796875,
      "learning_rate": 7.797840369314081e-07,
      "loss": 0.5313,
      "reward": 0.5546875,
      "reward_std": 0.17377189174294472,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5546875,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.171875,
      "epoch": 0.3989431968295905,
      "grad_norm": 10.47969913482666,
      "kl": 3.029296875,
      "learning_rate": 7.761992392908791e-07,
      "loss": 0.391,
      "reward": 0.78515625,
      "reward_std": 0.1711183786392212,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53515625,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 256.0,
      "epoch": 0.40158520475561427,
      "grad_norm": 27.210330963134766,
      "kl": 3.328125,
      "learning_rate": 7.725951295299005e-07,
      "loss": 0.8581,
      "reward": 1.56640625,
      "reward_std": 0.18129342049360275,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 608.375,
      "epoch": 0.404227212681638,
      "grad_norm": 7.893120765686035,
      "kl": 3.9453125,
      "learning_rate": 7.689720153550853e-07,
      "loss": 0.5819,
      "reward": 0.73828125,
      "reward_std": 0.13392486423254013,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.48828125,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.484375,
      "epoch": 0.40686922060766184,
      "grad_norm": 6.096236705780029,
      "kl": 3.18359375,
      "learning_rate": 7.653302060955789e-07,
      "loss": 0.4258,
      "reward": 1.078125,
      "reward_std": 0.20535630360245705,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.328125,
      "epoch": 0.4095112285336856,
      "grad_norm": 9.526097297668457,
      "kl": 3.87890625,
      "learning_rate": 7.616700126766492e-07,
      "loss": 0.6043,
      "reward": 1.05078125,
      "reward_std": 0.15629850327968597,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55078125,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.953125,
      "epoch": 0.41215323645970936,
      "grad_norm": 13.829514503479004,
      "kl": 4.109375,
      "learning_rate": 7.579917475931409e-07,
      "loss": 0.3873,
      "reward": 0.52734375,
      "reward_std": 0.18767033517360687,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.52734375,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 331.265625,
      "epoch": 0.4147952443857332,
      "grad_norm": 12.386381149291992,
      "kl": 3.4296875,
      "learning_rate": 7.54295724882796e-07,
      "loss": 0.7169,
      "reward": 1.328125,
      "reward_std": 0.2166232354938984,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 457.0,
      "epoch": 0.41743725231175693,
      "grad_norm": 7.208274841308594,
      "kl": 4.09375,
      "learning_rate": 7.505822600994423e-07,
      "loss": 0.6254,
      "reward": 1.28515625,
      "reward_std": 0.17519249208271503,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53515625,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.28125,
      "epoch": 0.4200792602377807,
      "grad_norm": 10.335708618164062,
      "kl": 4.54296875,
      "learning_rate": 7.468516702860519e-07,
      "loss": 0.5237,
      "reward": 0.51953125,
      "reward_std": 0.18916139006614685,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51953125,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 517.96875,
      "epoch": 0.4227212681638045,
      "grad_norm": 15.89622688293457,
      "kl": 3.72265625,
      "learning_rate": 7.43104273947674e-07,
      "loss": 0.3898,
      "reward": 1.01953125,
      "reward_std": 0.17299087904393673,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51953125,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 307.828125,
      "epoch": 0.42536327608982827,
      "grad_norm": 8.838927268981934,
      "kl": 2.689453125,
      "learning_rate": 7.393403910242418e-07,
      "loss": 0.4323,
      "reward": 1.02734375,
      "reward_std": 0.13064508698880672,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.52734375,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.3125,
      "epoch": 0.42800528401585203,
      "grad_norm": 15.761492729187012,
      "kl": 2.98828125,
      "learning_rate": 7.355603428632565e-07,
      "loss": 0.23,
      "reward": 1.3671875,
      "reward_std": 0.22000113874673843,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6171875,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 582.75,
      "epoch": 0.43064729194187584,
      "grad_norm": 14.52424144744873,
      "kl": 2.109375,
      "learning_rate": 7.317644521923526e-07,
      "loss": 0.5996,
      "reward": 0.7578125,
      "reward_std": 0.1417398639023304,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5078125,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 364.53125,
      "epoch": 0.4332892998678996,
      "grad_norm": 12.958600044250488,
      "kl": 1.623046875,
      "learning_rate": 7.279530430917441e-07,
      "loss": 0.0741,
      "reward": 0.796875,
      "reward_std": 0.1477682925760746,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.53125,
      "epoch": 0.43593130779392336,
      "grad_norm": 9.96493911743164,
      "kl": 1.7197265625,
      "learning_rate": 7.241264409665554e-07,
      "loss": 0.441,
      "reward": 0.82421875,
      "reward_std": 0.21464627608656883,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.57421875,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 582.5625,
      "epoch": 0.4385733157199472,
      "grad_norm": 7.888613224029541,
      "kl": 1.72265625,
      "learning_rate": 7.202849725190397e-07,
      "loss": 0.3068,
      "reward": 1.0078125,
      "reward_std": 0.17024145647883415,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5078125,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 534.1875,
      "epoch": 0.44121532364597094,
      "grad_norm": 8.195699691772461,
      "kl": 1.58642578125,
      "learning_rate": 7.16428965720686e-07,
      "loss": 0.3543,
      "reward": 0.8046875,
      "reward_std": 0.2195490226149559,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5546875,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 262.46875,
      "epoch": 0.4438573315719947,
      "grad_norm": 11.303885459899902,
      "kl": 0.970703125,
      "learning_rate": 7.125587497842189e-07,
      "loss": 0.4021,
      "reward": 0.80078125,
      "reward_std": 0.1908670738339424,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55078125,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 507.84375,
      "epoch": 0.4464993394980185,
      "grad_norm": 18.99937629699707,
      "kl": 1.3828125,
      "learning_rate": 7.086746551354895e-07,
      "loss": 0.5214,
      "reward": 0.76953125,
      "reward_std": 0.1896660476922989,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51953125,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 481.09375,
      "epoch": 0.44914134742404227,
      "grad_norm": 16.57875633239746,
      "kl": 1.5625,
      "learning_rate": 7.047770133852676e-07,
      "loss": 0.4899,
      "reward": 1.0546875,
      "reward_std": 0.19582437723875046,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5546875,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 434.171875,
      "epoch": 0.45178335535006603,
      "grad_norm": 6.7548298835754395,
      "kl": 1.451171875,
      "learning_rate": 7.008661573009273e-07,
      "loss": 0.3438,
      "reward": 1.30078125,
      "reward_std": 0.1738675981760025,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55078125,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.4375,
      "epoch": 0.45442536327608984,
      "grad_norm": 11.90649127960205,
      "kl": 1.791015625,
      "learning_rate": 6.969424207780374e-07,
      "loss": 0.1403,
      "reward": 1.3515625,
      "reward_std": 0.2295953370630741,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6015625,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 567.515625,
      "epoch": 0.4570673712021136,
      "grad_norm": 4.553245544433594,
      "kl": 2.3828125,
      "learning_rate": 6.930061388118557e-07,
      "loss": 0.4131,
      "reward": 1.05859375,
      "reward_std": 0.21736154332756996,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55859375,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 522.15625,
      "epoch": 0.45970937912813736,
      "grad_norm": 10.5054931640625,
      "kl": 2.76171875,
      "learning_rate": 6.890576474687263e-07,
      "loss": 0.2456,
      "reward": 0.76171875,
      "reward_std": 0.17176654934883118,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51171875,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 760.015625,
      "epoch": 0.4623513870541612,
      "grad_norm": 12.109650611877441,
      "kl": 4.10546875,
      "learning_rate": 6.850972838573888e-07,
      "loss": 0.4345,
      "reward": 0.7578125,
      "reward_std": 0.17381427809596062,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5078125,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 670.84375,
      "epoch": 0.46499339498018494,
      "grad_norm": 9.500724792480469,
      "kl": 3.11328125,
      "learning_rate": 6.811253861001961e-07,
      "loss": 0.448,
      "reward": 0.8125,
      "reward_std": 0.2038702666759491,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5625,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 668.578125,
      "epoch": 0.4676354029062087,
      "grad_norm": 3.1513185501098633,
      "kl": 2.83984375,
      "learning_rate": 6.771422933042477e-07,
      "loss": 0.4486,
      "reward": 0.7734375,
      "reward_std": 0.19701149314641953,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 460.640625,
      "epoch": 0.4702774108322325,
      "grad_norm": 3.928485631942749,
      "kl": 2.52734375,
      "learning_rate": 6.731483455324374e-07,
      "loss": 0.4601,
      "reward": 0.55078125,
      "reward_std": 0.1819697804749012,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55078125,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 500.734375,
      "epoch": 0.47291941875825627,
      "grad_norm": 5.9308905601501465,
      "kl": 2.89453125,
      "learning_rate": 6.691438837744191e-07,
      "loss": 0.5959,
      "reward": 1.0859375,
      "reward_std": 0.24082761257886887,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 318.859375,
      "epoch": 0.47556142668428003,
      "grad_norm": 8.880630493164062,
      "kl": 2.07373046875,
      "learning_rate": 6.651292499174959e-07,
      "loss": 0.2224,
      "reward": 1.0703125,
      "reward_std": 0.18467539176344872,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 504.203125,
      "epoch": 0.47820343461030385,
      "grad_norm": 7.29809045791626,
      "kl": 2.671875,
      "learning_rate": 6.611047867174298e-07,
      "loss": 0.5424,
      "reward": 0.796875,
      "reward_std": 0.19480633921921253,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 693.046875,
      "epoch": 0.4808454425363276,
      "grad_norm": 7.5113844871521,
      "kl": 3.078125,
      "learning_rate": 6.570708377691783e-07,
      "loss": 0.6193,
      "reward": 1.5859375,
      "reward_std": 0.2526575177907944,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 504.484375,
      "epoch": 0.48348745046235136,
      "grad_norm": 8.909899711608887,
      "kl": 2.7265625,
      "learning_rate": 6.530277474775602e-07,
      "loss": 0.572,
      "reward": 1.31640625,
      "reward_std": 0.20270539075136185,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 451.421875,
      "epoch": 0.4861294583883752,
      "grad_norm": 20.32670021057129,
      "kl": 2.5546875,
      "learning_rate": 6.489758610278509e-07,
      "loss": 0.4425,
      "reward": 1.08203125,
      "reward_std": 0.21750707924365997,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58203125,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 586.625,
      "epoch": 0.48877146631439894,
      "grad_norm": 6.589134693145752,
      "kl": 2.375,
      "learning_rate": 6.449155243563114e-07,
      "loss": 0.4211,
      "reward": 0.546875,
      "reward_std": 0.2208508811891079,
      "rewards/accuracy_reward": 0.015625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53125,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 534.640625,
      "epoch": 0.4914134742404227,
      "grad_norm": 9.064754486083984,
      "kl": 2.705078125,
      "learning_rate": 6.408470841206545e-07,
      "loss": 0.2999,
      "reward": 1.015625,
      "reward_std": 0.10510582849383354,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.515625,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 758.078125,
      "epoch": 0.4940554821664465,
      "grad_norm": 14.509212493896484,
      "kl": 3.865234375,
      "learning_rate": 6.367708876704476e-07,
      "loss": 0.494,
      "reward": 1.02734375,
      "reward_std": 0.20098446309566498,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.52734375,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.09375,
      "epoch": 0.4966974900924703,
      "grad_norm": 14.154923439025879,
      "kl": 2.201171875,
      "learning_rate": 6.326872830174566e-07,
      "loss": 0.1712,
      "reward": 1.0859375,
      "reward_std": 0.19368236511945724,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.234375,
      "epoch": 0.49933949801849403,
      "grad_norm": 17.86855125427246,
      "kl": 2.376953125,
      "learning_rate": 6.285966188059355e-07,
      "loss": 0.6533,
      "reward": 1.09375,
      "reward_std": 0.2263101488351822,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59375,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 512.125,
      "epoch": 0.5019815059445178,
      "grad_norm": 8.82755184173584,
      "kl": 3.04296875,
      "learning_rate": 6.244992442828585e-07,
      "loss": 0.3686,
      "reward": 0.7734375,
      "reward_std": 0.1519293300807476,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 560.03125,
      "epoch": 0.5046235138705416,
      "grad_norm": 15.707466125488281,
      "kl": 3.029296875,
      "learning_rate": 6.203955092681039e-07,
      "loss": 0.3194,
      "reward": 1.0703125,
      "reward_std": 0.1986095793545246,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 768.546875,
      "epoch": 0.5072655217965654,
      "grad_norm": 11.438809394836426,
      "kl": 2.88671875,
      "learning_rate": 6.162857641245869e-07,
      "loss": 0.6017,
      "reward": 1.28125,
      "reward_std": 0.21250617876648903,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53125,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.859375,
      "epoch": 0.5099075297225891,
      "grad_norm": 5.250596523284912,
      "kl": 1.41015625,
      "learning_rate": 6.12170359728347e-07,
      "loss": 0.2562,
      "reward": 1.33203125,
      "reward_std": 0.20339645817875862,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58203125,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 460.921875,
      "epoch": 0.512549537648613,
      "grad_norm": 8.758655548095703,
      "kl": 1.962890625,
      "learning_rate": 6.080496474385916e-07,
      "loss": 0.34,
      "reward": 0.79296875,
      "reward_std": 0.19175675138831139,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.54296875,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 426.734375,
      "epoch": 0.5151915455746368,
      "grad_norm": 13.022716522216797,
      "kl": 1.361328125,
      "learning_rate": 6.039239790676974e-07,
      "loss": 0.49,
      "reward": 1.1484375,
      "reward_std": 0.2307521291077137,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6484375,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 519.953125,
      "epoch": 0.5178335535006605,
      "grad_norm": 14.834174156188965,
      "kl": 2.318359375,
      "learning_rate": 5.997937068511754e-07,
      "loss": 0.1528,
      "reward": 1.06640625,
      "reward_std": 0.14010578021407127,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 568.75,
      "epoch": 0.5204755614266843,
      "grad_norm": 10.123536109924316,
      "kl": 2.12109375,
      "learning_rate": 5.956591834175964e-07,
      "loss": 0.5013,
      "reward": 1.31640625,
      "reward_std": 0.21957488358020782,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 658.484375,
      "epoch": 0.523117569352708,
      "grad_norm": 6.424520015716553,
      "kl": 3.1796875,
      "learning_rate": 5.915207617584858e-07,
      "loss": 0.4787,
      "reward": 1.3125,
      "reward_std": 0.22040452808141708,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5625,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 594.515625,
      "epoch": 0.5257595772787318,
      "grad_norm": 5.053133010864258,
      "kl": 2.666015625,
      "learning_rate": 5.873787951981868e-07,
      "loss": 0.4661,
      "reward": 0.75390625,
      "reward_std": 0.17793777957558632,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.50390625,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 339.03125,
      "epoch": 0.5284015852047557,
      "grad_norm": 4.2198944091796875,
      "kl": 1.740234375,
      "learning_rate": 5.832336373636933e-07,
      "loss": 0.3366,
      "reward": 1.28515625,
      "reward_std": 0.17389780096709728,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53515625,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 468.40625,
      "epoch": 0.5310435931307794,
      "grad_norm": 2.89648175239563,
      "kl": 1.6396484375,
      "learning_rate": 5.790856421544598e-07,
      "loss": 0.3048,
      "reward": 1.5859375,
      "reward_std": 0.19600137695670128,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 475.765625,
      "epoch": 0.5336856010568032,
      "grad_norm": 6.781806468963623,
      "kl": 2.189453125,
      "learning_rate": 5.749351637121865e-07,
      "loss": 0.3492,
      "reward": 0.828125,
      "reward_std": 0.20571819692850113,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 503.40625,
      "epoch": 0.5363276089828269,
      "grad_norm": 3.5012331008911133,
      "kl": 2.72265625,
      "learning_rate": 5.707825563905828e-07,
      "loss": 0.4152,
      "reward": 1.30078125,
      "reward_std": 0.17533257603645325,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55078125,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 374.390625,
      "epoch": 0.5389696169088507,
      "grad_norm": 16.517194747924805,
      "kl": 1.6416015625,
      "learning_rate": 5.666281747251153e-07,
      "loss": 0.4345,
      "reward": 1.2890625,
      "reward_std": 0.18729007616639137,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5390625,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 344.546875,
      "epoch": 0.5416116248348745,
      "grad_norm": 4.214947700500488,
      "kl": 1.6279296875,
      "learning_rate": 5.624723734027373e-07,
      "loss": 0.3469,
      "reward": 1.01171875,
      "reward_std": 0.1350011769682169,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51171875,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 640.53125,
      "epoch": 0.5442536327608983,
      "grad_norm": 4.432642936706543,
      "kl": 2.634765625,
      "learning_rate": 5.583155072316085e-07,
      "loss": 0.3449,
      "reward": 1.01953125,
      "reward_std": 0.14237725362181664,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51953125,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 346.4375,
      "epoch": 0.5468956406869221,
      "grad_norm": 6.426868915557861,
      "kl": 2.21875,
      "learning_rate": 5.541579311108009e-07,
      "loss": 0.4081,
      "reward": 1.33203125,
      "reward_std": 0.20600395277142525,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58203125,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 599.890625,
      "epoch": 0.5495376486129459,
      "grad_norm": 9.497568130493164,
      "kl": 2.8671875,
      "learning_rate": 5.5e-07,
      "loss": 0.594,
      "reward": 1.0390625,
      "reward_std": 0.2189657799899578,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5390625,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 587.703125,
      "epoch": 0.5521796565389696,
      "grad_norm": 2.5981221199035645,
      "kl": 2.576171875,
      "learning_rate": 5.458420688891992e-07,
      "loss": 0.3634,
      "reward": 1.34765625,
      "reward_std": 0.2173020839691162,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59765625,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 251.59375,
      "epoch": 0.5548216644649934,
      "grad_norm": 12.541109085083008,
      "kl": 1.94140625,
      "learning_rate": 5.416844927683916e-07,
      "loss": 0.482,
      "reward": 1.33984375,
      "reward_std": 0.22426774725317955,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58984375,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 353.359375,
      "epoch": 0.5574636723910171,
      "grad_norm": 21.176788330078125,
      "kl": 2.33203125,
      "learning_rate": 5.375276265972627e-07,
      "loss": 0.2879,
      "reward": 1.05078125,
      "reward_std": 0.18691154941916466,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55078125,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.1875,
      "epoch": 0.560105680317041,
      "grad_norm": 4.283320903778076,
      "kl": 2.701171875,
      "learning_rate": 5.333718252748849e-07,
      "loss": 0.3272,
      "reward": 1.5546875,
      "reward_std": 0.1786573100835085,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5546875,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.40625,
      "epoch": 0.5627476882430648,
      "grad_norm": 7.3552470207214355,
      "kl": 2.76953125,
      "learning_rate": 5.292174436094172e-07,
      "loss": 0.4091,
      "reward": 1.05859375,
      "reward_std": 0.19953873381018639,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55859375,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 599.421875,
      "epoch": 0.5653896961690885,
      "grad_norm": 7.531975746154785,
      "kl": 4.07421875,
      "learning_rate": 5.250648362878135e-07,
      "loss": 0.6474,
      "reward": 1.3359375,
      "reward_std": 0.22002986446022987,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 442.046875,
      "epoch": 0.5680317040951123,
      "grad_norm": 9.658491134643555,
      "kl": 2.96875,
      "learning_rate": 5.209143578455401e-07,
      "loss": 0.3931,
      "reward": 1.31640625,
      "reward_std": 0.21046040952205658,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 323.140625,
      "epoch": 0.570673712021136,
      "grad_norm": 16.756044387817383,
      "kl": 2.46484375,
      "learning_rate": 5.167663626363066e-07,
      "loss": 0.1497,
      "reward": 1.328125,
      "reward_std": 0.19799000024795532,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 485.28125,
      "epoch": 0.5733157199471598,
      "grad_norm": 14.802947998046875,
      "kl": 2.94921875,
      "learning_rate": 5.126212048018133e-07,
      "loss": 0.3226,
      "reward": 0.5546875,
      "reward_std": 0.17373281717300415,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5546875,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 553.296875,
      "epoch": 0.5759577278731837,
      "grad_norm": 6.547313213348389,
      "kl": 3.35546875,
      "learning_rate": 5.084792382415141e-07,
      "loss": 0.7209,
      "reward": 0.5703125,
      "reward_std": 0.20446551591157913,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 580.078125,
      "epoch": 0.5785997357992074,
      "grad_norm": 7.502042293548584,
      "kl": 2.875,
      "learning_rate": 5.043408165824037e-07,
      "loss": 0.522,
      "reward": 1.07421875,
      "reward_std": 0.2559613697230816,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.57421875,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 291.125,
      "epoch": 0.5812417437252312,
      "grad_norm": 9.088134765625,
      "kl": 1.806640625,
      "learning_rate": 5.002062931488247e-07,
      "loss": 0.5338,
      "reward": 0.8046875,
      "reward_std": 0.18990932404994965,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5546875,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 533.640625,
      "epoch": 0.583883751651255,
      "grad_norm": 11.220687866210938,
      "kl": 2.3984375,
      "learning_rate": 4.960760209323026e-07,
      "loss": 0.6041,
      "reward": 0.5234375,
      "reward_std": 0.19436774030327797,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 612.890625,
      "epoch": 0.5865257595772787,
      "grad_norm": 6.296652317047119,
      "kl": 3.07421875,
      "learning_rate": 4.919503525614086e-07,
      "loss": 0.5521,
      "reward": 0.76953125,
      "reward_std": 0.18084516376256943,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51953125,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 265.609375,
      "epoch": 0.5891677675033025,
      "grad_norm": 3.475614309310913,
      "kl": 1.50390625,
      "learning_rate": 4.878296402716531e-07,
      "loss": 0.2643,
      "reward": 1.38671875,
      "reward_std": 0.20747815072536469,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.63671875,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 714.1875,
      "epoch": 0.5918097754293263,
      "grad_norm": 6.395312786102295,
      "kl": 3.357421875,
      "learning_rate": 4.837142358754131e-07,
      "loss": 0.6176,
      "reward": 1.2734375,
      "reward_std": 0.21194355189800262,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.859375,
      "epoch": 0.5944517833553501,
      "grad_norm": 6.891757488250732,
      "kl": 2.8984375,
      "learning_rate": 4.79604490731896e-07,
      "loss": 0.42,
      "reward": 1.06640625,
      "reward_std": 0.2256414033472538,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 581.984375,
      "epoch": 0.5970937912813739,
      "grad_norm": 7.385695934295654,
      "kl": 3.4140625,
      "learning_rate": 4.755007557171414e-07,
      "loss": 0.6208,
      "reward": 1.05078125,
      "reward_std": 0.19489648565649986,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55078125,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 346.296875,
      "epoch": 0.5997357992073976,
      "grad_norm": 5.268566608428955,
      "kl": 2.427734375,
      "learning_rate": 4.7140338119406455e-07,
      "loss": 0.3306,
      "reward": 1.109375,
      "reward_std": 0.22719038277864456,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.609375,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 454.625,
      "epoch": 0.6023778071334214,
      "grad_norm": 11.538866996765137,
      "kl": 2.423828125,
      "learning_rate": 4.6731271698254326e-07,
      "loss": 0.664,
      "reward": 1.109375,
      "reward_std": 0.21347813308238983,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.609375,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.4375,
      "epoch": 0.6050198150594451,
      "grad_norm": 10.027405738830566,
      "kl": 2.166015625,
      "learning_rate": 4.632291123295524e-07,
      "loss": 0.3504,
      "reward": 1.3125,
      "reward_std": 0.2073436863720417,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5625,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 778.328125,
      "epoch": 0.607661822985469,
      "grad_norm": 8.903005599975586,
      "kl": 4.5234375,
      "learning_rate": 4.5915291587934547e-07,
      "loss": 0.6184,
      "reward": 1.0234375,
      "reward_std": 0.21458512544631958,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 496.296875,
      "epoch": 0.6103038309114928,
      "grad_norm": 19.55433464050293,
      "kl": 4.23046875,
      "learning_rate": 4.5508447564368856e-07,
      "loss": 0.6321,
      "reward": 1.33984375,
      "reward_std": 0.22301983460783958,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58984375,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 686.265625,
      "epoch": 0.6129458388375165,
      "grad_norm": 6.192388534545898,
      "kl": 3.7890625,
      "learning_rate": 4.510241389721493e-07,
      "loss": 0.5918,
      "reward": 1.5859375,
      "reward_std": 0.2616988569498062,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 540.578125,
      "epoch": 0.6155878467635403,
      "grad_norm": 7.43271017074585,
      "kl": 3.13671875,
      "learning_rate": 4.4697225252243976e-07,
      "loss": 0.6237,
      "reward": 1.3515625,
      "reward_std": 0.24065708369016647,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6015625,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 521.265625,
      "epoch": 0.618229854689564,
      "grad_norm": 7.898358345031738,
      "kl": 2.81640625,
      "learning_rate": 4.4292916223082165e-07,
      "loss": 0.5285,
      "reward": 1.3046875,
      "reward_std": 0.2356991246342659,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5546875,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 493.28125,
      "epoch": 0.6208718626155878,
      "grad_norm": 10.038056373596191,
      "kl": 2.90234375,
      "learning_rate": 4.388952132825701e-07,
      "loss": 0.2489,
      "reward": 1.140625,
      "reward_std": 0.2295135334134102,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.640625,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.984375,
      "epoch": 0.6235138705416117,
      "grad_norm": 4.178317546844482,
      "kl": 2.68359375,
      "learning_rate": 4.3487075008250397e-07,
      "loss": 0.4859,
      "reward": 0.79296875,
      "reward_std": 0.2021397091448307,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.54296875,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 698.25,
      "epoch": 0.6261558784676354,
      "grad_norm": 7.887820243835449,
      "kl": 4.2421875,
      "learning_rate": 4.3085611622558084e-07,
      "loss": 0.6169,
      "reward": 1.28125,
      "reward_std": 0.21125948429107666,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53125,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 548.328125,
      "epoch": 0.6287978863936592,
      "grad_norm": 5.685881614685059,
      "kl": 2.59375,
      "learning_rate": 4.268516544675628e-07,
      "loss": 0.3334,
      "reward": 1.0625,
      "reward_std": 0.20200148969888687,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5625,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.75,
      "epoch": 0.631439894319683,
      "grad_norm": 11.868870735168457,
      "kl": 2.0859375,
      "learning_rate": 4.228577066957522e-07,
      "loss": 0.1258,
      "reward": 1.3671875,
      "reward_std": 0.22833861783146858,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6171875,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 599.859375,
      "epoch": 0.6340819022457067,
      "grad_norm": 5.297094345092773,
      "kl": 2.8125,
      "learning_rate": 4.1887461389980394e-07,
      "loss": 0.3444,
      "reward": 1.046875,
      "reward_std": 0.22738776728510857,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 460.609375,
      "epoch": 0.6367239101717305,
      "grad_norm": 9.069931983947754,
      "kl": 2.166015625,
      "learning_rate": 4.149027161426113e-07,
      "loss": 0.5227,
      "reward": 1.34375,
      "reward_std": 0.21560321748256683,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59375,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 753.9375,
      "epoch": 0.6393659180977543,
      "grad_norm": 3.11356258392334,
      "kl": 2.849609375,
      "learning_rate": 4.1094235253127374e-07,
      "loss": 0.4795,
      "reward": 1.046875,
      "reward_std": 0.20162740349769592,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 626.859375,
      "epoch": 0.6420079260237781,
      "grad_norm": 4.849280834197998,
      "kl": 2.39453125,
      "learning_rate": 4.069938611881443e-07,
      "loss": 0.5037,
      "reward": 0.796875,
      "reward_std": 0.18199804052710533,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 552.296875,
      "epoch": 0.6446499339498019,
      "grad_norm": 5.1860456466674805,
      "kl": 2.4404296875,
      "learning_rate": 4.030575792219626e-07,
      "loss": 0.3665,
      "reward": 1.296875,
      "reward_std": 0.1943066604435444,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 638.5625,
      "epoch": 0.6472919418758256,
      "grad_norm": 9.586490631103516,
      "kl": 2.599609375,
      "learning_rate": 3.9913384269907293e-07,
      "loss": 0.2958,
      "reward": 1.33203125,
      "reward_std": 0.22680200263857841,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58203125,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 608.640625,
      "epoch": 0.6499339498018494,
      "grad_norm": 7.131601810455322,
      "kl": 2.166015625,
      "learning_rate": 3.952229866147323e-07,
      "loss": 0.2385,
      "reward": 1.375,
      "reward_std": 0.2418774701654911,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.625,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 661.015625,
      "epoch": 0.6525759577278731,
      "grad_norm": 5.848790645599365,
      "kl": 2.306640625,
      "learning_rate": 3.913253448645103e-07,
      "loss": 0.4711,
      "reward": 1.08203125,
      "reward_std": 0.22584940120577812,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58203125,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 531.96875,
      "epoch": 0.655217965653897,
      "grad_norm": 5.778437614440918,
      "kl": 1.859375,
      "learning_rate": 3.8744125021578123e-07,
      "loss": 0.3466,
      "reward": 1.2734375,
      "reward_std": 0.1622530035674572,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 525.40625,
      "epoch": 0.6578599735799208,
      "grad_norm": 3.1933047771453857,
      "kl": 1.833984375,
      "learning_rate": 3.835710342793139e-07,
      "loss": 0.2862,
      "reward": 1.30078125,
      "reward_std": 0.15551739931106567,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55078125,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 480.75,
      "epoch": 0.6605019815059445,
      "grad_norm": 8.949792861938477,
      "kl": 1.197265625,
      "learning_rate": 3.797150274809604e-07,
      "loss": 0.326,
      "reward": 1.3359375,
      "reward_std": 0.2217497080564499,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 428.203125,
      "epoch": 0.6631439894319683,
      "grad_norm": 3.1499345302581787,
      "kl": 1.2763671875,
      "learning_rate": 3.7587355903344466e-07,
      "loss": 0.1597,
      "reward": 0.875,
      "reward_std": 0.21982388943433762,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.625,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 369.6875,
      "epoch": 0.665785997357992,
      "grad_norm": 4.168592929840088,
      "kl": 1.3583984375,
      "learning_rate": 3.7204695690825593e-07,
      "loss": 0.1939,
      "reward": 1.28125,
      "reward_std": 0.1477414984256029,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53125,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 544.015625,
      "epoch": 0.6684280052840158,
      "grad_norm": 7.520803451538086,
      "kl": 1.921875,
      "learning_rate": 3.682355478076473e-07,
      "loss": 0.2638,
      "reward": 0.82421875,
      "reward_std": 0.2656807042658329,
      "rewards/accuracy_reward": 0.265625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55859375,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 561.34375,
      "epoch": 0.6710700132100397,
      "grad_norm": 6.172038555145264,
      "kl": 2.318359375,
      "learning_rate": 3.6443965713674354e-07,
      "loss": 0.3545,
      "reward": 1.02734375,
      "reward_std": 0.19002593867480755,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.52734375,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 521.8125,
      "epoch": 0.6737120211360634,
      "grad_norm": 6.321176528930664,
      "kl": 1.609375,
      "learning_rate": 3.606596089757583e-07,
      "loss": 0.3466,
      "reward": 1.58984375,
      "reward_std": 0.2514568492770195,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58984375,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 307.46875,
      "epoch": 0.6763540290620872,
      "grad_norm": 4.846172332763672,
      "kl": 1.09765625,
      "learning_rate": 3.5689572605232597e-07,
      "loss": 0.2335,
      "reward": 1.3359375,
      "reward_std": 0.20273161679506302,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 468.5,
      "epoch": 0.678996036988111,
      "grad_norm": 12.14126968383789,
      "kl": 1.138671875,
      "learning_rate": 3.531483297139481e-07,
      "loss": 0.1721,
      "reward": 0.80078125,
      "reward_std": 0.1630447916686535,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55078125,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 565.203125,
      "epoch": 0.6816380449141347,
      "grad_norm": 3.9592182636260986,
      "kl": 1.837890625,
      "learning_rate": 3.4941773990055777e-07,
      "loss": 0.2977,
      "reward": 1.10546875,
      "reward_std": 0.25015248730778694,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.60546875,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 480.484375,
      "epoch": 0.6842800528401585,
      "grad_norm": 9.579623222351074,
      "kl": 1.62109375,
      "learning_rate": 3.45704275117204e-07,
      "loss": 0.4312,
      "reward": 1.08203125,
      "reward_std": 0.24054544791579247,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58203125,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 496.796875,
      "epoch": 0.6869220607661823,
      "grad_norm": 4.918056964874268,
      "kl": 1.14306640625,
      "learning_rate": 3.4200825240685914e-07,
      "loss": 0.1878,
      "reward": 1.1015625,
      "reward_std": 0.22064152732491493,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6015625,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 515.96875,
      "epoch": 0.6895640686922061,
      "grad_norm": 11.338505744934082,
      "kl": 1.765625,
      "learning_rate": 3.3832998732335085e-07,
      "loss": 0.4868,
      "reward": 1.0859375,
      "reward_std": 0.21507646515965462,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 579.875,
      "epoch": 0.6922060766182299,
      "grad_norm": 10.862038612365723,
      "kl": 2.357421875,
      "learning_rate": 3.346697939044211e-07,
      "loss": 0.6303,
      "reward": 0.77734375,
      "reward_std": 0.20420579984784126,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.52734375,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 641.796875,
      "epoch": 0.6948480845442536,
      "grad_norm": 7.440125465393066,
      "kl": 2.716796875,
      "learning_rate": 3.310279846449147e-07,
      "loss": 0.5692,
      "reward": 0.83203125,
      "reward_std": 0.2302125133574009,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58203125,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 490.625,
      "epoch": 0.6974900924702774,
      "grad_norm": 11.042434692382812,
      "kl": 1.890625,
      "learning_rate": 3.2740487047009954e-07,
      "loss": 0.575,
      "reward": 0.8203125,
      "reward_std": 0.21583595871925354,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 535.375,
      "epoch": 0.7001321003963011,
      "grad_norm": 9.307427406311035,
      "kl": 1.8515625,
      "learning_rate": 3.23800760709121e-07,
      "loss": 0.2549,
      "reward": 1.0625,
      "reward_std": 0.19687864929437637,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5625,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 573.34375,
      "epoch": 0.702774108322325,
      "grad_norm": 4.253864765167236,
      "kl": 2.693359375,
      "learning_rate": 3.2021596306859195e-07,
      "loss": 0.4737,
      "reward": 0.8125,
      "reward_std": 0.1992315910756588,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5625,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 648.3125,
      "epoch": 0.7054161162483488,
      "grad_norm": 7.490243911743164,
      "kl": 3.2275390625,
      "learning_rate": 3.1665078360632254e-07,
      "loss": 0.377,
      "reward": 1.078125,
      "reward_std": 0.22863000631332397,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 489.296875,
      "epoch": 0.7080581241743725,
      "grad_norm": 4.917722702026367,
      "kl": 2.056640625,
      "learning_rate": 3.1310552670518987e-07,
      "loss": 0.3075,
      "reward": 1.12109375,
      "reward_std": 0.23855430632829666,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.62109375,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 450.921875,
      "epoch": 0.7107001321003963,
      "grad_norm": 3.3728554248809814,
      "kl": 2.087890625,
      "learning_rate": 3.0958049504715024e-07,
      "loss": 0.3534,
      "reward": 1.07421875,
      "reward_std": 0.20587731152772903,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.57421875,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 550.140625,
      "epoch": 0.71334214002642,
      "grad_norm": 6.581082344055176,
      "kl": 2.974609375,
      "learning_rate": 3.0607598958739777e-07,
      "loss": 0.3513,
      "reward": 1.08203125,
      "reward_std": 0.21218016743659973,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58203125,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 666.0625,
      "epoch": 0.7159841479524438,
      "grad_norm": 3.782729387283325,
      "kl": 3.47265625,
      "learning_rate": 3.0259230952866976e-07,
      "loss": 0.5161,
      "reward": 0.8515625,
      "reward_std": 0.266521442681551,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6015625,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 589.734375,
      "epoch": 0.7186261558784677,
      "grad_norm": 12.191798210144043,
      "kl": 2.857421875,
      "learning_rate": 2.991297522957015e-07,
      "loss": 0.257,
      "reward": 1.05859375,
      "reward_std": 0.1889869049191475,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55859375,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 476.515625,
      "epoch": 0.7212681638044914,
      "grad_norm": 5.739687442779541,
      "kl": 2.828125,
      "learning_rate": 2.9568861350983365e-07,
      "loss": 0.3424,
      "reward": 0.578125,
      "reward_std": 0.20889347046613693,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 606.921875,
      "epoch": 0.7239101717305152,
      "grad_norm": 8.41596794128418,
      "kl": 2.6015625,
      "learning_rate": 2.922691869637727e-07,
      "loss": 0.2616,
      "reward": 1.1171875,
      "reward_std": 0.24007226526737213,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6171875,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 491.59375,
      "epoch": 0.726552179656539,
      "grad_norm": 4.1023335456848145,
      "kl": 1.966796875,
      "learning_rate": 2.88871764596508e-07,
      "loss": 0.2751,
      "reward": 1.3515625,
      "reward_std": 0.2043364755809307,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6015625,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 565.84375,
      "epoch": 0.7291941875825627,
      "grad_norm": 5.3786540031433105,
      "kl": 2.720703125,
      "learning_rate": 2.854966364683872e-07,
      "loss": 0.3457,
      "reward": 0.828125,
      "reward_std": 0.20211807265877724,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 663.78125,
      "epoch": 0.7318361955085865,
      "grad_norm": 4.460934638977051,
      "kl": 3.201171875,
      "learning_rate": 2.821440907363516e-07,
      "loss": 0.4525,
      "reward": 0.8203125,
      "reward_std": 0.23223434761166573,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 641.265625,
      "epoch": 0.7344782034346103,
      "grad_norm": 16.07205581665039,
      "kl": 2.826171875,
      "learning_rate": 2.7881441362933464e-07,
      "loss": 0.334,
      "reward": 1.0625,
      "reward_std": 0.19014282897114754,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5625,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 676.0625,
      "epoch": 0.7371202113606341,
      "grad_norm": 11.935088157653809,
      "kl": 2.81640625,
      "learning_rate": 2.755078894238245e-07,
      "loss": 0.23,
      "reward": 0.78515625,
      "reward_std": 0.20001451671123505,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53515625,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 603.546875,
      "epoch": 0.7397622192866579,
      "grad_norm": 9.738125801086426,
      "kl": 2.033203125,
      "learning_rate": 2.722248004195932e-07,
      "loss": 0.2735,
      "reward": 1.09375,
      "reward_std": 0.20607677102088928,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59375,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 732.6875,
      "epoch": 0.7424042272126816,
      "grad_norm": 7.031618118286133,
      "kl": 2.41015625,
      "learning_rate": 2.689654269155955e-07,
      "loss": 0.2994,
      "reward": 0.82421875,
      "reward_std": 0.20312216132879257,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.57421875,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 578.875,
      "epoch": 0.7450462351387054,
      "grad_norm": 5.801688194274902,
      "kl": 1.40234375,
      "learning_rate": 2.657300471860372e-07,
      "loss": 0.2932,
      "reward": 1.05078125,
      "reward_std": 0.20492718927562237,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55078125,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 721.109375,
      "epoch": 0.7476882430647291,
      "grad_norm": 11.897012710571289,
      "kl": 2.43359375,
      "learning_rate": 2.625189374566175e-07,
      "loss": 0.5936,
      "reward": 0.7578125,
      "reward_std": 0.15211578272283077,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5078125,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 641.53125,
      "epoch": 0.750330250990753,
      "grad_norm": 5.453853130340576,
      "kl": 1.376953125,
      "learning_rate": 2.593323718809458e-07,
      "loss": 0.3039,
      "reward": 1.3671875,
      "reward_std": 0.2303219847381115,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6171875,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 596.28125,
      "epoch": 0.7529722589167768,
      "grad_norm": 5.665752410888672,
      "kl": 1.35546875,
      "learning_rate": 2.561706225171352e-07,
      "loss": 0.3616,
      "reward": 1.04296875,
      "reward_std": 0.17159553244709969,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.54296875,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 536.78125,
      "epoch": 0.7556142668428005,
      "grad_norm": 3.726806879043579,
      "kl": 1.5693359375,
      "learning_rate": 2.5303395930457494e-07,
      "loss": 0.2881,
      "reward": 1.3203125,
      "reward_std": 0.2022528052330017,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 495.984375,
      "epoch": 0.7582562747688243,
      "grad_norm": 3.6658847332000732,
      "kl": 1.1884765625,
      "learning_rate": 2.499226500408845e-07,
      "loss": 0.1181,
      "reward": 1.1171875,
      "reward_std": 0.1793758161365986,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6171875,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 859.984375,
      "epoch": 0.760898282694848,
      "grad_norm": 4.845893383026123,
      "kl": 2.955078125,
      "learning_rate": 2.4683696035904926e-07,
      "loss": 0.4852,
      "reward": 1.0078125,
      "reward_std": 0.1604960411787033,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5078125,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 440.8125,
      "epoch": 0.7635402906208718,
      "grad_norm": 2.4910755157470703,
      "kl": 0.85302734375,
      "learning_rate": 2.437771537047423e-07,
      "loss": 0.3161,
      "reward": 1.07421875,
      "reward_std": 0.2174788936972618,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.57421875,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 669.734375,
      "epoch": 0.7661822985468957,
      "grad_norm": 4.620151519775391,
      "kl": 1.90234375,
      "learning_rate": 2.407434913138318e-07,
      "loss": 0.3675,
      "reward": 0.5859375,
      "reward_std": 0.22324015572667122,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 531.953125,
      "epoch": 0.7688243064729194,
      "grad_norm": 11.40556526184082,
      "kl": 1.4501953125,
      "learning_rate": 2.377362321900777e-07,
      "loss": 0.0233,
      "reward": 1.36328125,
      "reward_std": 0.21594615280628204,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.61328125,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 696.34375,
      "epoch": 0.7714663143989432,
      "grad_norm": 3.5709707736968994,
      "kl": 1.853515625,
      "learning_rate": 2.3475563308301908e-07,
      "loss": 0.2536,
      "reward": 0.84765625,
      "reward_std": 0.20635812729597092,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59765625,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 604.40625,
      "epoch": 0.774108322324967,
      "grad_norm": 6.535892486572266,
      "kl": 1.3740234375,
      "learning_rate": 2.3180194846605364e-07,
      "loss": 0.1969,
      "reward": 1.1171875,
      "reward_std": 0.23528173938393593,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6171875,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 703.734375,
      "epoch": 0.7767503302509907,
      "grad_norm": 6.631422996520996,
      "kl": 2.017578125,
      "learning_rate": 2.288754305147115e-07,
      "loss": 0.3918,
      "reward": 1.296875,
      "reward_std": 0.20271231979131699,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 550.125,
      "epoch": 0.7793923381770145,
      "grad_norm": 5.805858612060547,
      "kl": 1.369140625,
      "learning_rate": 2.259763290851255e-07,
      "loss": 0.3276,
      "reward": 1.0625,
      "reward_std": 0.18768509849905968,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5625,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 776.859375,
      "epoch": 0.7820343461030383,
      "grad_norm": 6.1796135902404785,
      "kl": 2.36328125,
      "learning_rate": 2.231048916926992e-07,
      "loss": 0.2911,
      "reward": 1.3203125,
      "reward_std": 0.2180866338312626,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 459.90625,
      "epoch": 0.7846763540290621,
      "grad_norm": 4.840709686279297,
      "kl": 1.15234375,
      "learning_rate": 2.2026136349097495e-07,
      "loss": 0.2601,
      "reward": 0.86328125,
      "reward_std": 0.21641594916582108,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.61328125,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 626.34375,
      "epoch": 0.7873183619550859,
      "grad_norm": 4.876105308532715,
      "kl": 2.0615234375,
      "learning_rate": 2.1744598725070347e-07,
      "loss": 0.403,
      "reward": 1.28515625,
      "reward_std": 0.17794826440513134,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53515625,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 560.78125,
      "epoch": 0.7899603698811096,
      "grad_norm": 5.7457451820373535,
      "kl": 1.310546875,
      "learning_rate": 2.146590033391168e-07,
      "loss": 0.259,
      "reward": 1.32421875,
      "reward_std": 0.20343545079231262,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.57421875,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 666.5625,
      "epoch": 0.7926023778071334,
      "grad_norm": 4.766579627990723,
      "kl": 1.6201171875,
      "learning_rate": 2.11900649699407e-07,
      "loss": 0.1752,
      "reward": 1.109375,
      "reward_std": 0.2358247935771942,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.609375,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 631.734375,
      "epoch": 0.7952443857331571,
      "grad_norm": 3.2293262481689453,
      "kl": 1.62890625,
      "learning_rate": 2.0917116183041074e-07,
      "loss": 0.2575,
      "reward": 1.33984375,
      "reward_std": 0.22996815666556358,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58984375,
      "step": 301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 740.484375,
      "epoch": 0.797886393659181,
      "grad_norm": 3.1481125354766846,
      "kl": 2.294921875,
      "learning_rate": 2.0647077276650366e-07,
      "loss": 0.3915,
      "reward": 0.828125,
      "reward_std": 0.22289753332734108,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 472.625,
      "epoch": 0.8005284015852048,
      "grad_norm": 14.101240158081055,
      "kl": 1.4130859375,
      "learning_rate": 2.037997130577045e-07,
      "loss": 0.5247,
      "reward": 0.86328125,
      "reward_std": 0.24362235516309738,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.61328125,
      "step": 303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 579.9375,
      "epoch": 0.8031704095112285,
      "grad_norm": 2.720280885696411,
      "kl": 1.720703125,
      "learning_rate": 2.0115821074999156e-07,
      "loss": 0.2849,
      "reward": 1.3359375,
      "reward_std": 0.21295345574617386,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 592.1875,
      "epoch": 0.8058124174372523,
      "grad_norm": 4.275804042816162,
      "kl": 1.8828125,
      "learning_rate": 1.9854649136583307e-07,
      "loss": 0.3054,
      "reward": 1.09765625,
      "reward_std": 0.222886573523283,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59765625,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 468.53125,
      "epoch": 0.808454425363276,
      "grad_norm": 5.911637306213379,
      "kl": 1.4951171875,
      "learning_rate": 1.9596477788493254e-07,
      "loss": 0.2116,
      "reward": 1.109375,
      "reward_std": 0.2025398500263691,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.609375,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 548.859375,
      "epoch": 0.8110964332892999,
      "grad_norm": 5.387912273406982,
      "kl": 1.599609375,
      "learning_rate": 1.9341329072519176e-07,
      "loss": 0.351,
      "reward": 0.6171875,
      "reward_std": 0.22198385372757912,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6171875,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 573.5625,
      "epoch": 0.8137384412153237,
      "grad_norm": 5.202173709869385,
      "kl": 1.78125,
      "learning_rate": 1.9089224772389223e-07,
      "loss": 0.3517,
      "reward": 1.09375,
      "reward_std": 0.23804370686411858,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59375,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 652.28125,
      "epoch": 0.8163804491413474,
      "grad_norm": 4.832318305969238,
      "kl": 1.6396484375,
      "learning_rate": 1.884018641190968e-07,
      "loss": 0.2776,
      "reward": 1.69921875,
      "reward_std": 0.27570171654224396,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.69921875,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 653.1875,
      "epoch": 0.8190224570673712,
      "grad_norm": 5.5447211265563965,
      "kl": 2.037109375,
      "learning_rate": 1.8594235253127372e-07,
      "loss": 0.247,
      "reward": 1.046875,
      "reward_std": 0.21413858234882355,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 555.765625,
      "epoch": 0.821664464993395,
      "grad_norm": 10.55873966217041,
      "kl": 2.12109375,
      "learning_rate": 1.8351392294514326e-07,
      "loss": 0.4554,
      "reward": 1.2890625,
      "reward_std": 0.15378709696233273,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5390625,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 454.5625,
      "epoch": 0.8243064729194187,
      "grad_norm": 2.300844669342041,
      "kl": 1.0029296875,
      "learning_rate": 1.8111678269175055e-07,
      "loss": 0.1514,
      "reward": 1.11328125,
      "reward_std": 0.2071386780589819,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.61328125,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 672.796875,
      "epoch": 0.8269484808454426,
      "grad_norm": 5.112921237945557,
      "kl": 2.4970703125,
      "learning_rate": 1.78751136430764e-07,
      "loss": 0.4767,
      "reward": 1.078125,
      "reward_std": 0.20955145359039307,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 444.484375,
      "epoch": 0.8295904887714664,
      "grad_norm": 4.7589569091796875,
      "kl": 1.42578125,
      "learning_rate": 1.7641718613300228e-07,
      "loss": 0.2688,
      "reward": 0.640625,
      "reward_std": 0.238662201911211,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.640625,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 814.015625,
      "epoch": 0.8322324966974901,
      "grad_norm": 10.08535385131836,
      "kl": 3.3828125,
      "learning_rate": 1.7411513106319058e-07,
      "loss": 0.3937,
      "reward": 0.78125,
      "reward_std": 0.20346562936902046,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53125,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 613.03125,
      "epoch": 0.8348745046235139,
      "grad_norm": 12.75075912475586,
      "kl": 2.302734375,
      "learning_rate": 1.7184516776294832e-07,
      "loss": 0.2161,
      "reward": 0.8828125,
      "reward_std": 0.26399971544742584,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.015625,
      "rewards/tag_count_reward": 0.6171875,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 529.59375,
      "epoch": 0.8375165125495376,
      "grad_norm": 9.653738975524902,
      "kl": 1.8046875,
      "learning_rate": 1.6960749003400892e-07,
      "loss": 0.1588,
      "reward": 0.84375,
      "reward_std": 0.16583861783146858,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59375,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 583.625,
      "epoch": 0.8401585204755614,
      "grad_norm": 4.075193405151367,
      "kl": 1.640625,
      "learning_rate": 1.674022889216737e-07,
      "loss": 0.1898,
      "reward": 1.3125,
      "reward_std": 0.1740352250635624,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5625,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 669.0625,
      "epoch": 0.8428005284015853,
      "grad_norm": 4.472336292266846,
      "kl": 2.494140625,
      "learning_rate": 1.6522975269850104e-07,
      "loss": 0.3193,
      "reward": 0.85546875,
      "reward_std": 0.21766092255711555,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.60546875,
      "step": 319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 721.015625,
      "epoch": 0.845442536327609,
      "grad_norm": 6.250655174255371,
      "kl": 3.150390625,
      "learning_rate": 1.6309006684823239e-07,
      "loss": 0.5334,
      "reward": 1.0234375,
      "reward_std": 0.1688866000622511,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 675.921875,
      "epoch": 0.8480845442536328,
      "grad_norm": 1.8639191389083862,
      "kl": 2.427734375,
      "learning_rate": 1.6098341404995647e-07,
      "loss": 0.3932,
      "reward": 0.62890625,
      "reward_std": 0.24960599094629288,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.62890625,
      "step": 321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 600.421875,
      "epoch": 0.8507265521796565,
      "grad_norm": 4.137293338775635,
      "kl": 2.146484375,
      "learning_rate": 1.5890997416251224e-07,
      "loss": 0.351,
      "reward": 1.04296875,
      "reward_std": 0.1972101591527462,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.54296875,
      "step": 322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 565.4375,
      "epoch": 0.8533685601056803,
      "grad_norm": 10.063258171081543,
      "kl": 1.478515625,
      "learning_rate": 1.5686992420913372e-07,
      "loss": 0.0225,
      "reward": 0.86328125,
      "reward_std": 0.2034553661942482,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.61328125,
      "step": 323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 647.328125,
      "epoch": 0.8560105680317041,
      "grad_norm": 9.994471549987793,
      "kl": 2.05859375,
      "learning_rate": 1.5486343836233595e-07,
      "loss": 0.2504,
      "reward": 1.328125,
      "reward_std": 0.21247531473636627,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 690.234375,
      "epoch": 0.8586525759577279,
      "grad_norm": 9.103864669799805,
      "kl": 2.4921875,
      "learning_rate": 1.5289068792904495e-07,
      "loss": 0.483,
      "reward": 0.82421875,
      "reward_std": 0.2072843722999096,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.57421875,
      "step": 325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 732.96875,
      "epoch": 0.8612945838837517,
      "grad_norm": 7.12535285949707,
      "kl": 1.994140625,
      "learning_rate": 1.5095184133597217e-07,
      "loss": 0.4435,
      "reward": 1.08984375,
      "reward_std": 0.2667161263525486,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58984375,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.9375,
      "epoch": 0.8639365918097754,
      "grad_norm": 12.984781265258789,
      "kl": 1.0556640625,
      "learning_rate": 1.4904706411523448e-07,
      "loss": 0.3994,
      "reward": 1.32421875,
      "reward_std": 0.18335551768541336,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.57421875,
      "step": 327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 828.71875,
      "epoch": 0.8665785997357992,
      "grad_norm": 12.132417678833008,
      "kl": 1.8466796875,
      "learning_rate": 1.47176518890222e-07,
      "loss": 0.182,
      "reward": 1.0390625,
      "reward_std": 0.16892226040363312,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5390625,
      "step": 328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 822.609375,
      "epoch": 0.869220607661823,
      "grad_norm": 4.648046493530273,
      "kl": 2.0146484375,
      "learning_rate": 1.453403653617135e-07,
      "loss": 0.4329,
      "reward": 0.796875,
      "reward_std": 0.20767118781805038,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 590.765625,
      "epoch": 0.8718626155878467,
      "grad_norm": 3.794019937515259,
      "kl": 1.7001953125,
      "learning_rate": 1.4353876029424202e-07,
      "loss": 0.371,
      "reward": 1.09375,
      "reward_std": 0.216283418238163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59375,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 712.96875,
      "epoch": 0.8745046235138706,
      "grad_norm": 5.229684352874756,
      "kl": 2.5732421875,
      "learning_rate": 1.4177185750271055e-07,
      "loss": 0.3925,
      "reward": 1.09375,
      "reward_std": 0.23571135476231575,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59375,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 646.328125,
      "epoch": 0.8771466314398944,
      "grad_norm": 5.142683506011963,
      "kl": 2.0380859375,
      "learning_rate": 1.400398078392602e-07,
      "loss": 0.4217,
      "reward": 0.828125,
      "reward_std": 0.2310670204460621,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 580.5,
      "epoch": 0.8797886393659181,
      "grad_norm": 9.393284797668457,
      "kl": 1.46875,
      "learning_rate": 1.3834275918039055e-07,
      "loss": 0.3297,
      "reward": 1.33984375,
      "reward_std": 0.18817520886659622,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58984375,
      "step": 333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 584.625,
      "epoch": 0.8824306472919419,
      "grad_norm": 6.900231838226318,
      "kl": 1.951171875,
      "learning_rate": 1.3668085641433462e-07,
      "loss": 0.2931,
      "reward": 0.86328125,
      "reward_std": 0.2518454007804394,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.61328125,
      "step": 334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.796875,
      "epoch": 0.8850726552179656,
      "grad_norm": 9.600037574768066,
      "kl": 1.091796875,
      "learning_rate": 1.3505424142868897e-07,
      "loss": 0.3829,
      "reward": 1.41796875,
      "reward_std": 0.23616278544068336,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.66796875,
      "step": 335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 631.875,
      "epoch": 0.8877146631439894,
      "grad_norm": 5.003634929656982,
      "kl": 1.6171875,
      "learning_rate": 1.334630530982997e-07,
      "loss": 0.2516,
      "reward": 1.3046875,
      "reward_std": 0.21555107831954956,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5546875,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 614.03125,
      "epoch": 0.8903566710700133,
      "grad_norm": 16.881690979003906,
      "kl": 1.8984375,
      "learning_rate": 1.319074272734056e-07,
      "loss": 0.0975,
      "reward": 1.08984375,
      "reward_std": 0.19282393157482147,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58984375,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 666.0,
      "epoch": 0.892998678996037,
      "grad_norm": 5.620565414428711,
      "kl": 2.3154296875,
      "learning_rate": 1.303874967680399e-07,
      "loss": 0.2757,
      "reward": 1.62109375,
      "reward_std": 0.2326289601624012,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.62109375,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 476.203125,
      "epoch": 0.8956406869220608,
      "grad_norm": 5.114979267120361,
      "kl": 1.1298828125,
      "learning_rate": 1.289033913486914e-07,
      "loss": 0.1405,
      "reward": 1.0703125,
      "reward_std": 0.1810067780315876,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 713.8125,
      "epoch": 0.8982826948480845,
      "grad_norm": 3.9009175300598145,
      "kl": 2.587890625,
      "learning_rate": 1.2745523772322461e-07,
      "loss": 0.4324,
      "reward": 1.31640625,
      "reward_std": 0.1788315549492836,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 642.796875,
      "epoch": 0.9009247027741083,
      "grad_norm": 5.570927619934082,
      "kl": 1.9873046875,
      "learning_rate": 1.2604315953006266e-07,
      "loss": 0.34,
      "reward": 0.86328125,
      "reward_std": 0.24456297606229782,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.61328125,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 637.578125,
      "epoch": 0.9035667107001321,
      "grad_norm": 8.186066627502441,
      "kl": 1.923828125,
      "learning_rate": 1.2466727732763125e-07,
      "loss": 0.4781,
      "reward": 0.8671875,
      "reward_std": 0.23449090123176575,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6171875,
      "step": 342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 775.796875,
      "epoch": 0.9062087186261559,
      "grad_norm": 5.553122043609619,
      "kl": 3.125,
      "learning_rate": 1.2332770858406538e-07,
      "loss": 0.5849,
      "reward": 0.78515625,
      "reward_std": 0.21501468122005463,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.53515625,
      "step": 343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 445.453125,
      "epoch": 0.9088507265521797,
      "grad_norm": 4.708739757537842,
      "kl": 1.2822265625,
      "learning_rate": 1.220245676671809e-07,
      "loss": 0.1695,
      "reward": 1.078125,
      "reward_std": 0.15526169911026955,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.578125,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 752.78125,
      "epoch": 0.9114927344782034,
      "grad_norm": 3.9118199348449707,
      "kl": 1.9716796875,
      "learning_rate": 1.2075796583470984e-07,
      "loss": 0.3416,
      "reward": 1.06640625,
      "reward_std": 0.21211567521095276,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 651.375,
      "epoch": 0.9141347424042272,
      "grad_norm": 5.419198513031006,
      "kl": 2.326171875,
      "learning_rate": 1.1952801122480167e-07,
      "loss": 0.2937,
      "reward": 0.59765625,
      "reward_std": 0.2001628838479519,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59765625,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 662.203125,
      "epoch": 0.916776750330251,
      "grad_norm": 10.185606002807617,
      "kl": 2.2119140625,
      "learning_rate": 1.183348088467908e-07,
      "loss": 0.2272,
      "reward": 1.01171875,
      "reward_std": 0.15968638472259045,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.51171875,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 476.359375,
      "epoch": 0.9194187582562747,
      "grad_norm": 5.287563323974609,
      "kl": 1.537109375,
      "learning_rate": 1.1717846057223143e-07,
      "loss": 0.1921,
      "reward": 0.60546875,
      "reward_std": 0.22014086320996284,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.60546875,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 593.1875,
      "epoch": 0.9220607661822986,
      "grad_norm": 4.420534133911133,
      "kl": 1.7568359375,
      "learning_rate": 1.1605906512619983e-07,
      "loss": 0.3432,
      "reward": 1.3515625,
      "reward_std": 0.23761418834328651,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6015625,
      "step": 349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 709.6875,
      "epoch": 0.9247027741083224,
      "grad_norm": 4.137857437133789,
      "kl": 2.36328125,
      "learning_rate": 1.1497671807886567e-07,
      "loss": 0.3999,
      "reward": 1.0703125,
      "reward_std": 0.19854220747947693,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 447.515625,
      "epoch": 0.9273447820343461,
      "grad_norm": 5.883572578430176,
      "kl": 1.359375,
      "learning_rate": 1.139315118373326e-07,
      "loss": 0.3009,
      "reward": 0.859375,
      "reward_std": 0.21957654133439064,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.609375,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 535.09375,
      "epoch": 0.9299867899603699,
      "grad_norm": 9.422240257263184,
      "kl": 1.3564453125,
      "learning_rate": 1.1292353563774873e-07,
      "loss": 0.3162,
      "reward": 1.08984375,
      "reward_std": 0.22193554788827896,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58984375,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 603.21875,
      "epoch": 0.9326287978863936,
      "grad_norm": 4.772337913513184,
      "kl": 2.2646484375,
      "learning_rate": 1.1195287553768821e-07,
      "loss": 0.2438,
      "reward": 0.62890625,
      "reward_std": 0.28237032890319824,
      "rewards/accuracy_reward": 0.015625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.61328125,
      "step": 353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 714.09375,
      "epoch": 0.9352708058124174,
      "grad_norm": 9.603926658630371,
      "kl": 2.470703125,
      "learning_rate": 1.1101961440880352e-07,
      "loss": 0.3789,
      "reward": 1.05859375,
      "reward_std": 0.19248899817466736,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.55859375,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 628.40625,
      "epoch": 0.9379128137384413,
      "grad_norm": 16.06355857849121,
      "kl": 2.0009765625,
      "learning_rate": 1.1012383192975041e-07,
      "loss": 0.0823,
      "reward": 1.33203125,
      "reward_std": 0.18909762054681778,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58203125,
      "step": 355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 578.390625,
      "epoch": 0.940554821664465,
      "grad_norm": 3.9636921882629395,
      "kl": 1.8291015625,
      "learning_rate": 1.0926560457938536e-07,
      "loss": 0.2746,
      "reward": 1.3125,
      "reward_std": 0.2061732206493616,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5625,
      "step": 356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 520.203125,
      "epoch": 0.9431968295904888,
      "grad_norm": 6.897830486297607,
      "kl": 1.431640625,
      "learning_rate": 1.084450056302357e-07,
      "loss": 0.1525,
      "reward": 0.83203125,
      "reward_std": 0.21859385818243027,
      "rewards/accuracy_reward": 0.265625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 524.21875,
      "epoch": 0.9458388375165125,
      "grad_norm": 11.090557098388672,
      "kl": 1.40234375,
      "learning_rate": 1.0766210514224419e-07,
      "loss": 0.0591,
      "reward": 1.1328125,
      "reward_std": 0.23101669549942017,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6328125,
      "step": 358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 578.65625,
      "epoch": 0.9484808454425363,
      "grad_norm": 13.82530689239502,
      "kl": 2.111328125,
      "learning_rate": 1.0691696995678738e-07,
      "loss": 0.2682,
      "reward": 1.109375,
      "reward_std": 0.22573107481002808,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.609375,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 720.625,
      "epoch": 0.9511228533685601,
      "grad_norm": 6.005599021911621,
      "kl": 2.166015625,
      "learning_rate": 1.0620966369096884e-07,
      "loss": 0.3217,
      "reward": 1.34375,
      "reward_std": 0.2211884669959545,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59375,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 664.5625,
      "epoch": 0.9537648612945839,
      "grad_norm": 2.9504928588867188,
      "kl": 1.896484375,
      "learning_rate": 1.0554024673218806e-07,
      "loss": 0.3339,
      "reward": 1.31640625,
      "reward_std": 0.21037080883979797,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 719.90625,
      "epoch": 0.9564068692206077,
      "grad_norm": 3.942823886871338,
      "kl": 1.5712890625,
      "learning_rate": 1.0490877623298431e-07,
      "loss": 0.3399,
      "reward": 0.8515625,
      "reward_std": 0.23859936743974686,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6015625,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 497.296875,
      "epoch": 0.9590488771466315,
      "grad_norm": 11.69743824005127,
      "kl": 1.6708984375,
      "learning_rate": 1.0431530610615772e-07,
      "loss": 0.1801,
      "reward": 1.37109375,
      "reward_std": 0.20750074833631516,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.62109375,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 662.953125,
      "epoch": 0.9616908850726552,
      "grad_norm": 5.648345470428467,
      "kl": 2.005859375,
      "learning_rate": 1.0375988702016576e-07,
      "loss": 0.3905,
      "reward": 0.8203125,
      "reward_std": 0.21815017238259315,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 556.640625,
      "epoch": 0.964332892998679,
      "grad_norm": 3.6928138732910156,
      "kl": 1.544921875,
      "learning_rate": 1.0324256639479797e-07,
      "loss": 0.1847,
      "reward": 1.3359375,
      "reward_std": 0.2146303877234459,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5859375,
      "step": 365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 528.46875,
      "epoch": 0.9669749009247027,
      "grad_norm": 4.1989336013793945,
      "kl": 1.3134765625,
      "learning_rate": 1.0276338839712688e-07,
      "loss": 0.2739,
      "reward": 0.859375,
      "reward_std": 0.212420754134655,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.609375,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 807.125,
      "epoch": 0.9696169088507266,
      "grad_norm": 5.855282306671143,
      "kl": 2.8173828125,
      "learning_rate": 1.023223939377375e-07,
      "loss": 0.3144,
      "reward": 0.83203125,
      "reward_std": 0.2185688391327858,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58203125,
      "step": 367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 711.046875,
      "epoch": 0.9722589167767504,
      "grad_norm": 6.813151836395264,
      "kl": 1.77734375,
      "learning_rate": 1.0191962066723448e-07,
      "loss": 0.1714,
      "reward": 1.3203125,
      "reward_std": 0.18526797741651535,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5703125,
      "step": 368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 835.46875,
      "epoch": 0.9749009247027741,
      "grad_norm": 4.6733317375183105,
      "kl": 2.62109375,
      "learning_rate": 1.0155510297302745e-07,
      "loss": 0.4741,
      "reward": 0.7265625,
      "reward_std": 0.1361106839030981,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 501.609375,
      "epoch": 0.9775429326287979,
      "grad_norm": 7.580297946929932,
      "kl": 1.306640625,
      "learning_rate": 1.0122887197639539e-07,
      "loss": 0.106,
      "reward": 0.8828125,
      "reward_std": 0.21267065405845642,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.6328125,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 730.546875,
      "epoch": 0.9801849405548216,
      "grad_norm": 2.7990424633026123,
      "kl": 1.625,
      "learning_rate": 1.0094095552982936e-07,
      "loss": 0.1954,
      "reward": 1.06640625,
      "reward_std": 0.15350224822759628,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 634.125,
      "epoch": 0.9828269484808454,
      "grad_norm": 5.10625696182251,
      "kl": 1.578125,
      "learning_rate": 1.0069137821465474e-07,
      "loss": 0.3279,
      "reward": 1.59765625,
      "reward_std": 0.24609044939279556,
      "rewards/accuracy_reward": 1.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59765625,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 521.46875,
      "epoch": 0.9854689564068693,
      "grad_norm": 2.8827366828918457,
      "kl": 1.173828125,
      "learning_rate": 1.0048016133893242e-07,
      "loss": 0.2295,
      "reward": 0.81640625,
      "reward_std": 0.1789581961929798,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 528.515625,
      "epoch": 0.988110964332893,
      "grad_norm": 4.933093070983887,
      "kl": 1.3515625,
      "learning_rate": 1.0030732293563969e-07,
      "loss": 0.1593,
      "reward": 1.31640625,
      "reward_std": 0.18777159228920937,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.56640625,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 486.390625,
      "epoch": 0.9907529722589168,
      "grad_norm": 5.345139980316162,
      "kl": 1.306640625,
      "learning_rate": 1.0017287776113066e-07,
      "loss": 0.2942,
      "reward": 1.34765625,
      "reward_std": 0.23156387358903885,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.59765625,
      "step": 375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 832.59375,
      "epoch": 0.9933949801849405,
      "grad_norm": 5.978093147277832,
      "kl": 2.80859375,
      "learning_rate": 1.0007683729387628e-07,
      "loss": 0.562,
      "reward": 0.7734375,
      "reward_std": 0.20706837996840477,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 601.359375,
      "epoch": 0.9960369881109643,
      "grad_norm": 4.996700763702393,
      "kl": 1.537109375,
      "learning_rate": 1.0001920973348446e-07,
      "loss": 0.3616,
      "reward": 1.33984375,
      "reward_std": 0.2210528589785099,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.58984375,
      "step": 377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 651.0499954223633,
      "epoch": 0.9986789960369881,
      "grad_norm": 10.63793659210205,
      "kl": 1.486328125,
      "learning_rate": 1e-07,
      "loss": 0.16,
      "reward": 1.32421875,
      "reward_std": 0.1949087455868721,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.57421875,
      "step": 378
    },
    {
      "epoch": 0.9986789960369881,
      "step": 378,
      "total_flos": 0.0,
      "train_loss": 0.3501640140083889,
      "train_runtime": 20695.6892,
      "train_samples_per_second": 0.073,
      "train_steps_per_second": 0.018
    }
  ],
  "logging_steps": 1,
  "max_steps": 378,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 16,
  "trial_name": null,
  "trial_params": null
}